jflatt/snapchat-memories-downloader
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

file cleanup

Browse Source
This commit is contained in:
Josh Flatt 2025-12-10 22:57:43 -06:00
parent f110584be8
commit 832472d48d
3 changed files with 73 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
download_stats.py
View File

@ -30,7 +30,6 @@ def get_metadata(row: pd.Series) -> pd.Series:
"-createdate",
"-gpslatitude",
"-gpslongitude",
# *tags_to_check,
str(file_path),
]
result = subprocess.run(
@ -40,7 +39,6 @@ def get_metadata(row: pd.Series) -> pd.Series:
if len(metadata) < 3:
metadata = metadata + ["-"] * (3 - len(metadata))
# row["file"] = f"../downloads/" + str(Path.resolve(row["file"]))
row["createdate"] = metadata[0]
row["gpslatitude"] = metadata[1]
row["gpslongitude"] = metadata[2]
@ -50,7 +48,5 @@ def get_metadata(row: pd.Series) -> pd.Series:
df = file_df.apply(get_metadata, axis=1)
# df["file"] = f"../downloads/" + str(Path.resolve(df["file"])))
df.to_csv("./temp/filemetadata.csv", index=False, sep="#")

17
main3.py
View File

@ -27,7 +27,6 @@ def save_to_checkpoint(path: Path):
async def utc_filename(timestamp: str, url: str) -> Path:
# parse "YYYY-MM-DD HH:MM:SS UTC"
dt = datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S UTC")
dt = dt.replace(tzinfo=pytz.utc)
ext = "mp4" if ".mp4" in url.lower() else "jpg"
@ -59,17 +58,12 @@ async def download_one(session, sem, url, timestamp, failures, stats):
if path.exists():
return
# Already downloaded? (checkpoint resume)
if str(path) in stats["done"]:
return
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(data)
# record MB
stats["mb"] += len(data) / (1024 * 1024)
# checkpoint
save_to_checkpoint(path)
stats["done"].add(str(path))
@ -77,18 +71,17 @@ async def download_one(session, sem, url, timestamp, failures, stats):
except Exception as e:
if attempt == RETRIES:
failures.append((url, str(e)))
await asyncio.sleep(0.3 * attempt) # small backoff
await asyncio.sleep(0.3 * attempt)
async def main():
# Load JSON
with open("./resources/json/memories_history.json", "r") as f:
memories = json.load(f)["Saved Media"]
sem = asyncio.Semaphore(CONCURRENCY)
async with sem:
getting_urls = tqdm(total=len(memories), desc="Retrieving", unit="file")
# Prepare tasks
tasks_to_download = []
for item in memories:
@ -102,7 +95,7 @@ async def main():
stats = {
"mb": 0.0,
"done": load_checkpoint() # resume if exists
"done": load_checkpoint()
}
failures = []
@ -111,7 +104,6 @@ async def main():
async with aiohttp.ClientSession() as session:
start_time = time.time()
# tqdm based on number of files not yet done
to_process = [
# if not path.exists() and str(path) not in stats["done"]
(url, timestamp) for (url, timestamp) in tasks_to_download
@ -119,18 +111,15 @@ async def main():
progress = tqdm(total=len(to_process), desc="Downloading", unit="file")
# Wrap each task so progress bar updates *after* the download completes
async def wrapped(url, timestamp):
await download_one(session, sem, url, timestamp, failures, stats)
progress.update(1)
# Start tasks in batches
await asyncio.gather(*(wrapped(url, timestamp) for url, timestamp in to_process))
progress.close()
elapsed = time.time() - start_time
# Final summary
mb_total = stats["mb"]
speed = mb_total / elapsed if elapsed > 0 else 0

132
update_files.py
View File

@ -6,10 +6,10 @@ import os
from timezonefinder import TimezoneFinder
from datetime import datetime
import subprocess
from tqdm.asyncio import tqdm
df: pd.DataFrame = pd.read_json("./temp/file_info.json", orient="records")
df.head(5)
def get_join_col(row: pd.Series) -> pd.Series:
@ -27,12 +27,10 @@ def extract_lat_long(row: pd.Series) -> pd.Series:
df = df.apply(get_join_col, axis=1)
df = df.apply(extract_lat_long, axis=1)
df.head(5)
metadata = pd.read_csv("./temp/filemetadata.csv", sep="#")
metadata["file"] = f"./" + metadata["file"]
metadata.head(5)
def convert_lat_long_to_decimal_degrees(row: pd.Series) -> pd.Series:
@ -79,11 +77,10 @@ def get_file_name(row: pd.Series) -> pd.Series:
meta = metadata.apply(convert_lat_long_to_decimal_degrees, axis=1)
meta = meta.apply(get_file_name, axis=1)
meta.head(5)
def get_datetime_for_df(row: pd.Series) -> pd.Series:
from datetime import datetime
try:
row["createdate_mod"] = None
@ -158,25 +155,16 @@ def check_for_errors(row: pd.Series) -> pd.Series:
df1 = df0.apply(check_for_errors, axis=1)
errors = df1[df1["errors"] > 0]
errors.head(5)
print(f"Errors: {errors.shape[0]}")
img_errors = errors[errors["correct_media_type"] == "Image"]
print(f"Image errors: {img_errors.shape[0]}")
video_errors = errors[errors["correct_media_type"] == "Video"]
print(f"Video errors: {video_errors.shape[0]}")
video_errors.head(5)
def fix_filetype(row: pd.Series) -> pd.Series:
try:
if pd.isna(row["path"]) or row["path"] == "-":
@ -206,12 +194,6 @@ def fix_filetype(row: pd.Series) -> pd.Series:
return row
# wrong_filetype = df1[df1["correct_media_type"] != df1["actual_media_type"]]
# wrong_filetype = wrong_filetype.fillna("-")
# wrong_filetype.head(5)
# wrong_filetype.apply(fix_filetype, axis=1)
df1.apply(fix_filetype, axis=1)
@ -229,7 +211,7 @@ def get_localized_dt_and_offset(
if not tz_name or latitude == 0 or longitude == 0:
print(
f"⚠️ Warning: No timezone found for {latitude}, {longitude}. Defaulting to MDT/MST."
f"⚠️ Warning: No timezone found for {utc_dt.strftime("%Y-%m-%d %H:%M:%S")} UTC. Defaulting to MDT/MST."
)
local_tz = mountain_timezone
else:
@ -237,20 +219,21 @@ def get_localized_dt_and_offset(
dt_local = utc_dt.astimezone(local_tz)
# 'YYYY:MM:DD HH:MM:SS'
dt_str = dt_local.strftime("%Y:%m:%d %H:%M:%S")
# '+/-HHMM' -> '+/-HH:MM' (needed format for tags)
# Tags require offset in ±HH:MM format
offset_str: str = dt_local.strftime("%z")
if len(offset_str) == 5:
offset_str = offset_str[:3] + ":" + offset_str[3:]
return dt_str, offset_str
from tqdm.asyncio import tqdm
progress_bar = tqdm(
total=df1.shape[0],
desc="Updating EXIF",
unit="file",
disable=False,
)
def update_exif_with_exiftool(row: pd.Series) -> None:
progress_bar.update(1)
@ -271,7 +254,7 @@ def update_exif_with_exiftool(row: pd.Series) -> None:
lat_ref = "N" if latitude >= 0 else "S"
lon_ref = "E" if longitude >= 0 else "W"
dt_str, offset_str = get_localized_dt_and_offset(
local_dt, dynamic_tz = get_localized_dt_and_offset(
utc_dt=dt_utc,
latitude=row["correct_latitude"],
longitude=row["correct_longitude"],
@ -283,44 +266,52 @@ def update_exif_with_exiftool(row: pd.Series) -> None:
jpg_command = [
"exiftool",
# Local time
f"-IFD0:DateTime={dt_str}",
f"-ExifIFD:DateTimeOriginal={dt_str}",
f"-ExifIFD:DateTimeDigitized={dt_str}",
# Dynamic offset
f"-OffsetTimeOriginal={offset_str}",
# GPS tags
# Date/Time tags
f"-IFD0:DateTime={local_dt}",
f"-ExifIFD:DateTimeOriginal={local_dt}",
f"-ExifIFD:DateTimeDigitized={local_dt}",
f"-OffsetTimeOriginal={dynamic_tz}",
# Location tags
f"-GPSLatitude={latitude}",
f"-GPSLongitude={longitude}",
f"-GPSLatitudeRef='{lat_ref}'",
f"-GPSLongitudeRef='{lon_ref}'",
# Keep overwrite and target file
"-overwrite_original",
image_path,
]
jpg_command_no_gps = [
"exiftool",
# Date/Time tags
f"-IFD0:DateTime={local_dt}",
f"-ExifIFD:DateTimeOriginal={local_dt}",
f"-ExifIFD:DateTimeDigitized={local_dt}",
f"-OffsetTimeOriginal={dynamic_tz}",
"-overwrite_original",
image_path,
]
mp4_command = [
"exiftool",
# Date/Time tags
# EXIF Tags
f"-IFD0:DateTime={dt_str}",
f"-ExifIFD:DateTimeOriginal={dt_str}",
f"-ExifIFD:DateTimeDigitized={dt_str}",
f"-OffsetTimeOriginal={offset_str}",
f"-IFD0:DateTime={local_dt}",
f"-ExifIFD:DateTimeOriginal={local_dt}",
f"-ExifIFD:DateTimeDigitized={local_dt}",
f"-OffsetTimeOriginal={dynamic_tz}",
# QuickTime tags (must be set to UTC time)
f"-QuickTime:CreationDate={dt_utc_str}",
f"-QuickTime:TrackCreateDate={dt_utc_str}",
f"-QuickTime:MediaCreateDate={dt_utc_str}",
# This is for the display TimeZone of the above tags
# Not 100% sure needed, doesn't seem to hurt
f"-QuickTime:TimeZone={offset_str}",
# Composite Tag (Needed for location on Google Photos)
f"-QuickTime:TimeZone={dynamic_tz}",
# Location tags
# Needed for location on Google Photos
f"-QuickTime:GPSCoordinates={abs_latitude} {lat_ref}, {abs_longitude} {lon_ref}",
# XMP Tags (Adobe/Google compatibility)
# XMP Tags (Adobe/Google
f"-xmp:gpslatitude={latitude}",
f"-xmp:gpslongitude={longitude}",
# QuickTime Tags (Apple/iOS compatibility)
# QuickTime Tags (Apple/iOS)
f"-QuickTime:LocationLatitude={latitude}",
f"-QuickTime:LocationLongitude={longitude}",
# # Generic GPS tags (For broad compatibility)
# # Generic GPS tags (Broader compatibility)
f"-GPSLatitude={latitude}",
f"-GPSLongitude={longitude}",
f"-GPSLatitudeRef={lat_ref}",
@ -328,16 +319,41 @@ def update_exif_with_exiftool(row: pd.Series) -> None:
"-overwrite_original",
image_path,
]
mp4_command_no_gps = [
"exiftool",
# EXIF Tags
f"-IFD0:DateTime={local_dt}",
f"-ExifIFD:DateTimeOriginal={local_dt}",
f"-ExifIFD:DateTimeDigitized={local_dt}",
f"-OffsetTimeOriginal={dynamic_tz}",
# QuickTime tags (must be set to UTC time)
f"-QuickTime:CreationDate={dt_utc_str}",
f"-QuickTime:TrackCreateDate={dt_utc_str}",
f"-QuickTime:MediaCreateDate={dt_utc_str}",
f"-QuickTime:TimeZone={dynamic_tz}",
"-overwrite_original",
image_path,
]
try:
if file_type == "Video":
result = subprocess.run(
mp4_command, check=True, capture_output=True, text=True
)
if latitude == 0.0 and longitude == 0.0:
result = subprocess.run(
mp4_command_no_gps, check=True, capture_output=True, text=True
)
else:
result = subprocess.run(
mp4_command, check=True, capture_output=True, text=True
)
elif file_type == "Image":
result = subprocess.run(
jpg_command, check=True, capture_output=True, text=True
)
if latitude == 0.0 and longitude == 0.0:
result = subprocess.run(
jpg_command_no_gps, check=True, capture_output=True, text=True
)
else:
result = subprocess.run(
jpg_command, check=True, capture_output=True, text=True
)
else:
print(f"⚠️ Warning: Unknown media type for {image_path}, skipping.")
return
@ -356,14 +372,6 @@ def update_exif_with_exiftool(row: pd.Series) -> None:
)
progress_bar = tqdm(
total=df1.shape[0],
desc="Updating EXIF",
unit="file",
disable=False,
)
# img_errors.apply(update_exif_with_exiftool, axis=1) # type: ignore
df1.apply(update_exif_with_exiftool, axis=1) # type: ignore
progress_bar.close()