jflatt/snapchat-memories-downloader
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bad changes

Browse Source
This commit is contained in:
Josh Flatt 2025-12-10 16:25:46 -06:00
parent af8c3edc96
commit bd586a9541
9 changed files with 5741 additions and 3839 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
download_stats.py
View File

@ -1,15 +1,23 @@
import subprocess
from tqdm.asyncio import tqdm
from pathlib import Path
import pandas as pd
file_directory = Path("../downloads")
file_directory = Path("./downloads")
files_to_check = [
p
for p in file_directory.iterdir()
if p.is_file() and p.suffix.lower() in [".jpg", ".mp4"]
]
progress_bar = tqdm(
total=len(files_to_check),
desc="Gathering",
unit="file",
disable=False,
)
file_df = pd.DataFrame(data=files_to_check, columns=["file"])
@ -32,13 +40,17 @@ def get_metadata(row: pd.Series) -> pd.Series:
if len(metadata) < 3:
metadata = metadata + ["-"] * (3 - len(metadata))
# row["file"] = f"../downloads/" + str(Path.resolve(row["file"]))
row["createdate"] = metadata[0]
row["gpslatitude"] = metadata[1]
row["gpslongitude"] = metadata[2]
progress_bar.update(1)
return row
df = file_df.apply(get_metadata, axis=1)
# df["file"] = f"../downloads/" + str(Path.resolve(df["file"])))
df.to_csv("../temp/filemetadata.csv", index=False, sep="#")
df.to_csv("./temp/filemetadata.csv", index=False, sep="#")

8
main.py
View File

@ -165,11 +165,11 @@ async def download_memory(
timestamp = memory.date.timestamp()
os.utime(output_path, (timestamp, timestamp))
if add_exif and ext == ".jpg":
add_exif_data(output_path, memory)
# if add_exif and ext == ".jpg":
# add_exif_data(output_path, memory)
elif add_exif and ext == ".mp4":
add_metadata_mp4(output_path, memory)
# elif add_exif and ext == ".mp4":
# add_metadata_mp4(output_path, memory)
return True, len(response.content)
except Exception as e:

139
main3.py Normal file
View File

@ -0,0 +1,139 @@
import asyncio
import aiohttp
import json
import pytz
import time
import os
from pathlib import Path
from datetime import datetime
from tqdm import tqdm
CONCURRENCY = 50
RETRIES = 3
OUTPUT_DIR = Path("./downloads")
CHECKPOINT = Path("./temp/checkpoint.txt")
def load_checkpoint():
if CHECKPOINT.exists():
return set(CHECKPOINT.read_text().splitlines())
return set()
def save_to_checkpoint(path: Path):
with CHECKPOINT.open("a") as f:
f.write(str(path) + "\n")
# def utc_filename(timestamp: str, url: str) -> Path:
# dt = datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
# ext = "mp4" if ".mp4" in url.lower() else "jpg"
# return OUTPUT_DIR / f"{dt.strftime('%Y-%m-%d_%H-%M-%S')}.{ext}"
def utc_filename(timestamp: str, url: str) -> Path:
# parse "YYYY-MM-DD HH:MM:SS UTC"
dt = datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S UTC")
dt = dt.replace(tzinfo=pytz.utc)
ext = "mp4" if ".mp4" in url.lower() else "jpg"
out_path = os.path.join("./downloads",f"{dt.strftime('%Y-%m-%d_%H-%M-%S')}.{ext}")
print(out_path)
return Path(out_path)
async def download_one(session, sem, url, path, failures, stats):
if path.exists():
return
# Already downloaded? (checkpoint resume)
if str(path) in stats["done"]:
return
async with sem:
for attempt in range(1, RETRIES + 1):
try:
async with session.get(url) as resp:
resp.raise_for_status()
data = await resp.read()
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(data)
# record MB
stats["mb"] += len(data) / (1024 * 1024)
# checkpoint
save_to_checkpoint(path)
stats["done"].add(str(path))
return
except Exception as e:
if attempt == RETRIES:
failures.append((url, str(e)))
await asyncio.sleep(0.3 * attempt) # small backoff
async def main():
# Load JSON
with open("./resources/json/memories_history.json", "r") as f:
memories = json.load(f)["Saved Media"]
# Prepare tasks
tasks_to_download = []
for item in memories:
url = item.get("Download Link")
ts = item.get("Date")
if not url or not ts:
continue
outpath = utc_filename(ts, url)
tasks_to_download.append((url, outpath))
stats = {
"mb": 0.0,
"done": load_checkpoint() # resume if exists
}
failures = []
sem = asyncio.Semaphore(CONCURRENCY)
async with aiohttp.ClientSession() as session:
start_time = time.time()
# tqdm based on number of files not yet done
to_process = [
(url, path) for (url, path) in tasks_to_download
if not path.exists() and str(path) not in stats["done"]
]
progress = tqdm(total=len(to_process), desc="Downloading", unit="file")
# Wrap each task so progress bar updates *after* the download completes
async def wrapped(url, path):
await download_one(session, sem, url, path, failures, stats)
progress.update(1)
# Start tasks in batches
await asyncio.gather(*(wrapped(url, path) for url, path in to_process))
progress.close()
elapsed = time.time() - start_time
# Final summary
mb_total = stats["mb"]
speed = mb_total / elapsed if elapsed > 0 else 0
print("\n" + "=" * 60)
print(f"Downloaded: {len(to_process) - len(failures)} files")
print(f"Failed: {len(failures)} files")
print(f"Data: {mb_total:.2f} MB")
print(f"Speed: {speed:.2f} MB/s")
print("=" * 60)
if failures:
print("\n⚠️ Failed downloads:")
for url, err in failures:
print(f" - {url} ({err})")
print()
if __name__ == "__main__":
asyncio.run(main())

1143
notebooks/make_df.ipynb
View File

File diff suppressed because it is too large Load Diff

170
notebooks/verify_metadata.ipynb
View File

@ -65,43 +65,43 @@
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>../downloads/2019-04-05_23-27-17.jpg</td>\n",
" <td>../downloads/2025-03-15_01-17-10.mp4</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>../downloads/2024-10-23_00-29-09.jpg</td>\n",
" <td>../downloads/2025-02-28_18-25-46.mp4</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>../downloads/2025-02-16_00-09-41.mp4</td>\n",
" <td>../downloads/2024-09-09_23-21-56.jpg</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>../downloads/2023-03-10_22-11-18.jpg</td>\n",
" <td>../downloads/2025-02-18_13-40-28.mp4</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>../downloads/2024-02-24_22-18-07.jpg</td>\n",
" <td>../downloads/2024-10-18_02-04-34.mp4</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>../downloads/2020-10-24_03-27-27.mp4</td>\n",
" <td>../downloads/2025-03-08_06-20-39.mp4</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>../downloads/2020-11-04_04-40-23.mp4</td>\n",
" <td>../downloads/2025-06-28_02-54-42.jpg</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>../downloads/2017-06-09_22-44-18.jpg</td>\n",
" <td>../downloads/2025-04-13_22-54-57.jpg</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>../downloads/2018-12-26_06-49-23.mp4</td>\n",
" <td>../downloads/2025-03-04_19-48-15.mp4</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>../downloads/2024-07-25_04-57-56.mp4</td>\n",
" <td>../downloads/2025-04-28_21-29-43.mp4</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
@ -109,16 +109,16 @@
],
"text/plain": [
" file\n",
"0 ../downloads/2019-04-05_23-27-17.jpg\n",
"1 ../downloads/2024-10-23_00-29-09.jpg\n",
"2 ../downloads/2025-02-16_00-09-41.mp4\n",
"3 ../downloads/2023-03-10_22-11-18.jpg\n",
"4 ../downloads/2024-02-24_22-18-07.jpg\n",
"5 ../downloads/2020-10-24_03-27-27.mp4\n",
"6 ../downloads/2020-11-04_04-40-23.mp4\n",
"7 ../downloads/2017-06-09_22-44-18.jpg\n",
"8 ../downloads/2018-12-26_06-49-23.mp4\n",
"9 ../downloads/2024-07-25_04-57-56.mp4"
"0 ../downloads/2025-03-15_01-17-10.mp4\n",
"1 ../downloads/2025-02-28_18-25-46.mp4\n",
"2 ../downloads/2024-09-09_23-21-56.jpg\n",
"3 ../downloads/2025-02-18_13-40-28.mp4\n",
"4 ../downloads/2024-10-18_02-04-34.mp4\n",
"5 ../downloads/2025-03-08_06-20-39.mp4\n",
"6 ../downloads/2025-06-28_02-54-42.jpg\n",
"7 ../downloads/2025-04-13_22-54-57.jpg\n",
"8 ../downloads/2025-03-04_19-48-15.mp4\n",
"9 ../downloads/2025-04-28_21-29-43.mp4"
]
},
"execution_count": 3,
@ -175,7 +175,7 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 6,
"id": "e4389a3e",
"metadata": {},
"outputs": [
@ -209,105 +209,105 @@
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>../downloads/2019-04-05_23-27-17.jpg</td>\n",
" <td>2019:04:05 18:27:17</td>\n",
" <td>40 deg 24' 14.76\" N</td>\n",
" <td>104 deg 44' 21.52\" W</td>\n",
" <td>../downloads/2025-03-15_01-17-10.mp4</td>\n",
" <td>2025:03:15 01:17:29</td>\n",
" <td>-</td>\n",
" <td>-</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>../downloads/2024-10-23_00-29-09.jpg</td>\n",
" <td>2024:10:22 19:29:09</td>\n",
" <td>44 deg 6' 15.77\" N</td>\n",
" <td>94 deg 13' 10.52\" W</td>\n",
" <td>../downloads/2025-02-28_18-25-46.mp4</td>\n",
" <td>2025:02:28 18:26:16</td>\n",
" <td>-</td>\n",
" <td>-</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>../downloads/2025-02-16_00-09-41.mp4</td>\n",
" <td>2025:02:15 18:09:41</td>\n",
" <td>44 deg 21' 17.71\" N</td>\n",
" <td>93 deg 58' 1.20\" W</td>\n",
" <td>../downloads/2024-09-09_23-21-56.jpg</td>\n",
" <td>-</td>\n",
" <td>-</td>\n",
" <td>-</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>../downloads/2023-03-10_22-11-18.jpg</td>\n",
" <td>2023:03:10 16:11:18</td>\n",
" <td>40 deg 24' 39.13\" N</td>\n",
" <td>104 deg 41' 33.90\" W</td>\n",
" <td>../downloads/2025-02-18_13-40-28.mp4</td>\n",
" <td>2025:02:18 13:41:28</td>\n",
" <td>-</td>\n",
" <td>-</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>../downloads/2024-02-24_22-18-07.jpg</td>\n",
" <td>2024:02:24 16:18:07</td>\n",
" <td>40 deg 25' 17.08\" N</td>\n",
" <td>104 deg 45' 22.75\" W</td>\n",
" <td>../downloads/2024-10-18_02-04-34.mp4</td>\n",
" <td>2024:10:18 02:04:52</td>\n",
" <td>-</td>\n",
" <td>-</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>../downloads/2020-10-24_03-27-27.mp4</td>\n",
" <td>2020:10:23 21:27:27</td>\n",
" <td>40 deg 24' 37.62\" N</td>\n",
" <td>104 deg 42' 28.73\" W</td>\n",
" <td>../downloads/2025-03-08_06-20-39.mp4</td>\n",
" <td>2025:03:08 06:21:26</td>\n",
" <td>-</td>\n",
" <td>-</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>../downloads/2020-11-04_04-40-23.mp4</td>\n",
" <td>2020:11:03 21:40:23</td>\n",
" <td>40 deg 25' 55.63\" N</td>\n",
" <td>104 deg 46' 10.16\" W</td>\n",
" <td>../downloads/2025-06-28_02-54-42.jpg</td>\n",
" <td>-</td>\n",
" <td>-</td>\n",
" <td>-</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>../downloads/2017-06-09_22-44-18.jpg</td>\n",
" <td>2017:06:09 17:44:18</td>\n",
" <td>39 deg 50' 49.74\" N</td>\n",
" <td>104 deg 40' 29.57\" W</td>\n",
" <td>../downloads/2025-04-13_22-54-57.jpg</td>\n",
" <td>-</td>\n",
" <td>-</td>\n",
" <td>-</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>../downloads/2018-12-26_06-49-23.mp4</td>\n",
" <td>2018:12:25 23:49:23</td>\n",
" <td>40 deg 25' 56.21\" N</td>\n",
" <td>104 deg 46' 11.24\" W</td>\n",
" <td>../downloads/2025-03-04_19-48-15.mp4</td>\n",
" <td>2025:03:04 19:48:37</td>\n",
" <td>-</td>\n",
" <td>-</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>../downloads/2024-07-25_04-57-56.mp4</td>\n",
" <td>2024:07:24 23:57:56</td>\n",
" <td>44 deg 21' 17.17\" N</td>\n",
" <td>93 deg 57' 57.82\" W</td>\n",
" <td>../downloads/2025-04-28_21-29-43.mp4</td>\n",
" <td>2025:04:28 21:30:00</td>\n",
" <td>-</td>\n",
" <td>-</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" file createdate \\\n",
"0 ../downloads/2019-04-05_23-27-17.jpg 2019:04:05 18:27:17 \n",
"1 ../downloads/2024-10-23_00-29-09.jpg 2024:10:22 19:29:09 \n",
"2 ../downloads/2025-02-16_00-09-41.mp4 2025:02:15 18:09:41 \n",
"3 ../downloads/2023-03-10_22-11-18.jpg 2023:03:10 16:11:18 \n",
"4 ../downloads/2024-02-24_22-18-07.jpg 2024:02:24 16:18:07 \n",
"5 ../downloads/2020-10-24_03-27-27.mp4 2020:10:23 21:27:27 \n",
"6 ../downloads/2020-11-04_04-40-23.mp4 2020:11:03 21:40:23 \n",
"7 ../downloads/2017-06-09_22-44-18.jpg 2017:06:09 17:44:18 \n",
"8 ../downloads/2018-12-26_06-49-23.mp4 2018:12:25 23:49:23 \n",
"9 ../downloads/2024-07-25_04-57-56.mp4 2024:07:24 23:57:56 \n",
" file createdate gpslatitude \\\n",
"0 ../downloads/2025-03-15_01-17-10.mp4 2025:03:15 01:17:29 - \n",
"1 ../downloads/2025-02-28_18-25-46.mp4 2025:02:28 18:26:16 - \n",
"2 ../downloads/2024-09-09_23-21-56.jpg - - \n",
"3 ../downloads/2025-02-18_13-40-28.mp4 2025:02:18 13:41:28 - \n",
"4 ../downloads/2024-10-18_02-04-34.mp4 2024:10:18 02:04:52 - \n",
"5 ../downloads/2025-03-08_06-20-39.mp4 2025:03:08 06:21:26 - \n",
"6 ../downloads/2025-06-28_02-54-42.jpg - - \n",
"7 ../downloads/2025-04-13_22-54-57.jpg - - \n",
"8 ../downloads/2025-03-04_19-48-15.mp4 2025:03:04 19:48:37 - \n",
"9 ../downloads/2025-04-28_21-29-43.mp4 2025:04:28 21:30:00 - \n",
"\n",
" gpslatitude gpslongitude \n",
"0 40 deg 24' 14.76\" N 104 deg 44' 21.52\" W \n",
"1 44 deg 6' 15.77\" N 94 deg 13' 10.52\" W \n",
"2 44 deg 21' 17.71\" N 93 deg 58' 1.20\" W \n",
"3 40 deg 24' 39.13\" N 104 deg 41' 33.90\" W \n",
"4 40 deg 25' 17.08\" N 104 deg 45' 22.75\" W \n",
"5 40 deg 24' 37.62\" N 104 deg 42' 28.73\" W \n",
"6 40 deg 25' 55.63\" N 104 deg 46' 10.16\" W \n",
"7 39 deg 50' 49.74\" N 104 deg 40' 29.57\" W \n",
"8 40 deg 25' 56.21\" N 104 deg 46' 11.24\" W \n",
"9 44 deg 21' 17.17\" N 93 deg 57' 57.82\" W "
" gpslongitude \n",
"0 - \n",
"1 - \n",
"2 - \n",
"3 - \n",
"4 - \n",
"5 - \n",
"6 - \n",
"7 - \n",
"8 - \n",
"9 - "
]
},
"execution_count": 8,
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
@ -343,7 +343,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.13.11"
"version": "3.12.6"
}
},
"nbformat": 4,

1
requirements.txt
View File

@ -5,6 +5,7 @@ tqdm>=4.67.1
pillow
piexif
timezonefinder
aiohttp
# For development
jupyter

4023
temp/filemetadata.csv
View File

File diff suppressed because it is too large Load Diff

3711
temp/filemetadata0.csv Normal file
View File

File diff suppressed because it is too large Load Diff

369
update_files.py Normal file
View File

@ -0,0 +1,369 @@
import pandas as pd
import re
import pytz
import os
from timezonefinder import TimezoneFinder
from datetime import datetime
import subprocess
df: pd.DataFrame = pd.read_json("./temp/file_info.json", orient="records")
df.head(5)
def get_join_col(row: pd.Series) -> pd.Series:
row["file_dt"] = f"{row['Date'].strftime('%Y-%m-%d_%H-%M-%S')}"
return row
def extract_lat_long(row: pd.Series) -> pd.Series:
coordinates = row["Location"].split(": ")[1]
lat_str, long_str = coordinates.split(", ")
row["location_latitude"] = round(float(lat_str), 5)
row["location_longitude"] = round(float(long_str), 5)
return row
df = df.apply(get_join_col, axis=1)
df = df.apply(extract_lat_long, axis=1)
df.head(5)
metadata = pd.read_csv("./temp/filemetadata.csv", sep="#")
metadata["file"] = f"./" + metadata["file"]
metadata.head(5)
def convert_lat_long_to_decimal_degrees(row: pd.Series) -> pd.Series:
def dms_to_decimal(dms_string):
match = re.search(
r"(\d+)\s*deg\s*(\d+)'\s*([\d.]+)\"\s*([NSEW])", dms_string, re.IGNORECASE
)
if not match:
raise ValueError(f"Coordinate format not recognized: {dms_string}")
degrees = float(match.group(1))
minutes = float(match.group(2))
seconds = float(match.group(3))
direction = match.group(4).upper()
decimal_degrees = degrees + (minutes / 60) + (seconds / 3600)
if direction in ["S", "W"]:
decimal_degrees *= -1
decimal_degrees = round(decimal_degrees, 5)
return decimal_degrees
def convert(value: str) -> float:
if pd.isna(value):
return 0.0
if value == "-":
return 0.0
try:
return dms_to_decimal(value)
except ValueError:
return 0.0
row["Latitude"] = convert(row["gpslatitude"])
row["Longitude"] = convert(row["gpslongitude"])
return row
def get_file_name(row: pd.Series) -> pd.Series:
path_parts = row["file"].split("/")
file = path_parts[-1]
row["filename"] = file
row["extension"] = file.split(".")[-1].lower()
row["file_dt"] = file.split(".")[0]
return row
meta = metadata.apply(convert_lat_long_to_decimal_degrees, axis=1)
meta = meta.apply(get_file_name, axis=1)
meta.head(5)
def get_datetime_for_df(row: pd.Series) -> pd.Series:
from datetime import datetime
try:
row["createdate_mod"] = None
if row["createdate"] != "-":
createdate = datetime.strptime(row["createdate"], "%Y:%m:%d %H:%M:%S")
createdate_wtz = createdate.astimezone(pytz.timezone("America/Chicago"))
row["createdate_mod"] = createdate_wtz
date = datetime.strptime(row["file_dt"], "%Y-%m-%d_%H-%M-%S")
dt_w_tz = date.astimezone(pytz.UTC)
row["datetime"] = dt_w_tz
row["dt_join"] = f"{dt_w_tz}"
except ValueError:
row["createdate_mod"] = None
row["datetime"] = None
row["dt_join"] = ""
return row
def get_media_type(row: pd.Series) -> pd.Series:
row["file_type"] = "Unknown"
if row["extension"] == "jpg":
row["file_type"] = "Image"
if row["extension"] == "mp4":
row["file_type"] = "Video"
return row
data = meta.apply(get_datetime_for_df, axis=1)
data = data.apply(get_media_type, axis=1)
joined = pd.merge(
df,
data,
how="left",
left_on="file_dt",
right_on="file_dt",
suffixes=("_df", "_meta"),
)
col_mapper = {
"Date": "correct_date_utc",
"Media Type": "correct_media_type",
"location_latitude": "correct_latitude",
"location_longitude": "correct_longitude",
"file": "path",
"filename": "filename",
"createdate": "actual_date_cst",
"Latitude": "actual_latitude",
"Longitude": "actual_longitude",
"file_type": "actual_media_type",
}
df0 = joined[col_mapper.keys()].rename(columns=col_mapper)
def check_for_errors(row: pd.Series) -> pd.Series:
errors = 0
if row["correct_media_type"] != row["actual_media_type"]:
errors += 1
if row["actual_latitude"] == 0.00 and row["correct_media_type"] == "Image":
if row["correct_latitude"] != 0.00:
errors += 1
if abs(row["actual_longitude"] - row["correct_longitude"]) > 2.00:
errors += 1
if row["actual_date_cst"] == "-":
errors += 1
if pd.isna(row["path"]):
errors += 1
row["errors"] = errors
return row
df1 = df0.apply(check_for_errors, axis=1)
errors = df1[df1["errors"] > 0]
errors.head(5)
print(f"Errors: {errors.shape[0]}")
img_errors = errors[errors["correct_media_type"] == "Image"]
print(f"Image errors: {img_errors.shape[0]}")
video_errors = errors[errors["correct_media_type"] == "Video"]
print(f"Video errors: {video_errors.shape[0]}")
video_errors.head(5)
def fix_filetype(row: pd.Series) -> pd.Series:
try:
if pd.isna(row["path"]) or row["path"] == "-":
return row
mp4_file_path = f"./downloads/{row['filename'][:-4]}.mp4"
jpg_file_path = f"./downloads/{row['filename'][:-4]}.jpg"
if row["correct_media_type"] == row["actual_media_type"]:
return row
if row["path"] == "-":
return row
if row["correct_media_type"] == "Image":
os.rename(src=mp4_file_path, dst=jpg_file_path)
if os.path.exists(jpg_file_path):
row["filename"] = f"{row['filename'][:-4]}.jpg"
row["actual_media_type"] = "Image"
elif row["correct_media_type"] == "Video":
os.rename(src=jpg_file_path, dst=mp4_file_path)
if os.path.exists(mp4_file_path):
row["filename"] = f"{row['filename'][:-4]}.mp4"
row["actual_media_type"] = "Video"
except FileNotFoundError:
print(f"File not found: {row['filename']}")
return row
# wrong_filetype = df1[df1["correct_media_type"] != df1["actual_media_type"]]
# wrong_filetype = wrong_filetype.fillna("-")
# wrong_filetype.head(5)
# wrong_filetype.apply(fix_filetype, axis=1)
df1.apply(fix_filetype, axis=1)
tf = TimezoneFinder()
def get_localized_dt_and_offset(
utc_dt: datetime, latitude: float, longitude: float
) -> tuple[str, str]:
tz_name = tf.timezone_at(lng=longitude, lat=latitude)
utc_timezone = pytz.utc
central_timezone = pytz.timezone("America/Chicago")
mountain_timezone = pytz.timezone("America/Denver")
if not tz_name or latitude == 0 or longitude == 0:
print(
f"⚠️ Warning: No timezone found for {latitude}, {longitude}. Defaulting to MDT/MST."
)
local_tz = mountain_timezone
else:
local_tz = pytz.timezone(tz_name)
dt_local = utc_dt.astimezone(local_tz)
# 'YYYY:MM:DD HH:MM:SS'
dt_str = dt_local.strftime("%Y:%m:%d %H:%M:%S")
# '+/-HHMM' -> '+/-HH:MM' (needed format for tags)
offset_str: str = dt_local.strftime("%z")
if len(offset_str) == 5:
offset_str = offset_str[:3] + ":" + offset_str[3:]
return dt_str, offset_str
from tqdm.asyncio import tqdm
def update_exif_with_exiftool(row: pd.Series) -> None:
progress_bar.update(1)
image_path = row["path"]
dt_utc = row["correct_date_utc"] # timezone-aware, UTC datetime
latitude = row["correct_latitude"]
longitude = row["correct_longitude"]
file_type = row["correct_media_type"]
abs_latitude = abs(latitude)
abs_longitude = abs(longitude)
dt_utc_str = dt_utc.strftime("%Y:%m:%d %H:%M:%S")
if image_path is None or pd.isna(image_path) or image_path == "":
print(f"❌ File not found, skipping: {image_path}")
return
lat_ref = "N" if latitude >= 0 else "S"
lon_ref = "E" if longitude >= 0 else "W"
dt_str, offset_str = get_localized_dt_and_offset(
utc_dt=dt_utc,
latitude=row["correct_latitude"],
longitude=row["correct_longitude"],
)
if not os.path.exists(image_path):
print(f"❌ File not found, skipping: {image_path}")
return
jpg_command = [
"exiftool",
# Local time
f"-IFD0:DateTime={dt_str}",
f"-ExifIFD:DateTimeOriginal={dt_str}",
f"-ExifIFD:DateTimeDigitized={dt_str}",
# Dynamic offset
f"-OffsetTimeOriginal={offset_str}",
# GPS tags
f"-GPSLatitude={latitude}",
f"-GPSLongitude={longitude}",
f"-GPSLatitudeRef='{lat_ref}'",
f"-GPSLongitudeRef='{lon_ref}'",
# Keep overwrite and target file
"-overwrite_original",
image_path,
]
mp4_command = [
"exiftool",
# EXIF Tags
f"-IFD0:DateTime={dt_str}",
f"-ExifIFD:DateTimeOriginal={dt_str}",
f"-ExifIFD:DateTimeDigitized={dt_str}",
f"-OffsetTimeOriginal={offset_str}",
# QuickTime tags (must be set to UTC time)
f"-QuickTime:CreationDate={dt_utc_str}",
f"-QuickTime:TrackCreateDate={dt_utc_str}",
f"-QuickTime:MediaCreateDate={dt_utc_str}",
# This is for the display TimeZone of the above tags
# Not 100% sure needed, doesn't seem to hurt
f"-QuickTime:TimeZone={offset_str}",
# Composite Tag (Needed for location on Google Photos)
f"-QuickTime:GPSCoordinates={abs_latitude} {lat_ref}, {abs_longitude} {lon_ref}",
# XMP Tags (Adobe/Google compatibility)
f"-xmp:gpslatitude={latitude}",
f"-xmp:gpslongitude={longitude}",
# QuickTime Tags (Apple/iOS compatibility)
f"-QuickTime:LocationLatitude={latitude}",
f"-QuickTime:LocationLongitude={longitude}",
# # Generic GPS tags (For broad compatibility)
f"-GPSLatitude={latitude}",
f"-GPSLongitude={longitude}",
f"-GPSLatitudeRef={lat_ref}",
f"-GPSLongitudeRef={lon_ref}",
"-overwrite_original",
image_path,
]
try:
if file_type == "Video":
result = subprocess.run(
mp4_command, check=True, capture_output=True, text=True
)
elif file_type == "Image":
result = subprocess.run(
jpg_command, check=True, capture_output=True, text=True
)
else:
print(f"⚠️ Warning: Unknown media type for {image_path}, skipping.")
return
if "1 image files updated" in result.stdout:
pass
# print(f"✅ Updated: {image_path}")
else:
print(f"⚠️ Warning updating {image_path}: {result.stderr.strip()}")
except subprocess.CalledProcessError as e:
print(f"🛑 Subprocess Error on {image_path}: {e.stderr.strip()}")
except FileNotFoundError:
print(
"🛑 ERROR: ExifTool command not found. Is ExifTool installed and in your PATH?"
)
progress_bar = tqdm(
total=df1.shape[0],
desc="Updating EXIF",
unit="file",
disable=False,
)
# img_errors.apply(update_exif_with_exiftool, axis=1) # type: ignore
df1.apply(update_exif_with_exiftool, axis=1) # type: ignore