diff --git a/download_stats.py b/download_stats.py index beb4d2d..cd9dfac 100644 --- a/download_stats.py +++ b/download_stats.py @@ -30,7 +30,6 @@ def get_metadata(row: pd.Series) -> pd.Series: "-createdate", "-gpslatitude", "-gpslongitude", - # *tags_to_check, str(file_path), ] result = subprocess.run( @@ -40,7 +39,6 @@ def get_metadata(row: pd.Series) -> pd.Series: if len(metadata) < 3: metadata = metadata + ["-"] * (3 - len(metadata)) - # row["file"] = f"../downloads/" + str(Path.resolve(row["file"])) row["createdate"] = metadata[0] row["gpslatitude"] = metadata[1] row["gpslongitude"] = metadata[2] @@ -50,7 +48,5 @@ def get_metadata(row: pd.Series) -> pd.Series: df = file_df.apply(get_metadata, axis=1) -# df["file"] = f"../downloads/" + str(Path.resolve(df["file"]))) - df.to_csv("./temp/filemetadata.csv", index=False, sep="#") diff --git a/main3.py b/main3.py index 4873491..7e2f031 100644 --- a/main3.py +++ b/main3.py @@ -27,7 +27,6 @@ def save_to_checkpoint(path: Path): async def utc_filename(timestamp: str, url: str) -> Path: - # parse "YYYY-MM-DD HH:MM:SS UTC" dt = datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S UTC") dt = dt.replace(tzinfo=pytz.utc) ext = "mp4" if ".mp4" in url.lower() else "jpg" @@ -59,17 +58,12 @@ async def download_one(session, sem, url, timestamp, failures, stats): if path.exists(): return - # Already downloaded? (checkpoint resume) if str(path) in stats["done"]: return path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(data) - - # record MB stats["mb"] += len(data) / (1024 * 1024) - - # checkpoint save_to_checkpoint(path) stats["done"].add(str(path)) @@ -77,18 +71,17 @@ async def download_one(session, sem, url, timestamp, failures, stats): except Exception as e: if attempt == RETRIES: failures.append((url, str(e))) - await asyncio.sleep(0.3 * attempt) # small backoff + await asyncio.sleep(0.3 * attempt) async def main(): - # Load JSON with open("./resources/json/memories_history.json", "r") as f: memories = json.load(f)["Saved Media"] sem = asyncio.Semaphore(CONCURRENCY) async with sem: getting_urls = tqdm(total=len(memories), desc="Retrieving", unit="file") - # Prepare tasks + tasks_to_download = [] for item in memories: @@ -102,7 +95,7 @@ async def main(): stats = { "mb": 0.0, - "done": load_checkpoint() # resume if exists + "done": load_checkpoint() } failures = [] @@ -111,7 +104,6 @@ async def main(): async with aiohttp.ClientSession() as session: start_time = time.time() - # tqdm based on number of files not yet done to_process = [ # if not path.exists() and str(path) not in stats["done"] (url, timestamp) for (url, timestamp) in tasks_to_download @@ -119,18 +111,15 @@ async def main(): progress = tqdm(total=len(to_process), desc="Downloading", unit="file") - # Wrap each task so progress bar updates *after* the download completes async def wrapped(url, timestamp): await download_one(session, sem, url, timestamp, failures, stats) progress.update(1) - # Start tasks in batches await asyncio.gather(*(wrapped(url, timestamp) for url, timestamp in to_process)) progress.close() elapsed = time.time() - start_time - # Final summary mb_total = stats["mb"] speed = mb_total / elapsed if elapsed > 0 else 0 diff --git a/update_files.py b/update_files.py index 3716931..acc104d 100644 --- a/update_files.py +++ b/update_files.py @@ -6,10 +6,10 @@ import os from timezonefinder import TimezoneFinder from datetime import datetime import subprocess +from tqdm.asyncio import tqdm df: pd.DataFrame = pd.read_json("./temp/file_info.json", orient="records") -df.head(5) def get_join_col(row: pd.Series) -> pd.Series: @@ -27,12 +27,10 @@ def extract_lat_long(row: pd.Series) -> pd.Series: df = df.apply(get_join_col, axis=1) df = df.apply(extract_lat_long, axis=1) -df.head(5) metadata = pd.read_csv("./temp/filemetadata.csv", sep="#") metadata["file"] = f"./" + metadata["file"] -metadata.head(5) def convert_lat_long_to_decimal_degrees(row: pd.Series) -> pd.Series: @@ -79,11 +77,10 @@ def get_file_name(row: pd.Series) -> pd.Series: meta = metadata.apply(convert_lat_long_to_decimal_degrees, axis=1) meta = meta.apply(get_file_name, axis=1) -meta.head(5) def get_datetime_for_df(row: pd.Series) -> pd.Series: - from datetime import datetime + try: row["createdate_mod"] = None @@ -158,25 +155,16 @@ def check_for_errors(row: pd.Series) -> pd.Series: df1 = df0.apply(check_for_errors, axis=1) - errors = df1[df1["errors"] > 0] -errors.head(5) - - print(f"Errors: {errors.shape[0]}") - img_errors = errors[errors["correct_media_type"] == "Image"] print(f"Image errors: {img_errors.shape[0]}") - video_errors = errors[errors["correct_media_type"] == "Video"] print(f"Video errors: {video_errors.shape[0]}") -video_errors.head(5) - - def fix_filetype(row: pd.Series) -> pd.Series: try: if pd.isna(row["path"]) or row["path"] == "-": @@ -206,12 +194,6 @@ def fix_filetype(row: pd.Series) -> pd.Series: return row -# wrong_filetype = df1[df1["correct_media_type"] != df1["actual_media_type"]] -# wrong_filetype = wrong_filetype.fillna("-") -# wrong_filetype.head(5) - - -# wrong_filetype.apply(fix_filetype, axis=1) df1.apply(fix_filetype, axis=1) @@ -229,7 +211,7 @@ def get_localized_dt_and_offset( if not tz_name or latitude == 0 or longitude == 0: print( - f"⚠️ Warning: No timezone found for {latitude}, {longitude}. Defaulting to MDT/MST." + f"⚠️ Warning: No timezone found for {utc_dt.strftime("%Y-%m-%d %H:%M:%S")} UTC. Defaulting to MDT/MST." ) local_tz = mountain_timezone else: @@ -237,20 +219,21 @@ def get_localized_dt_and_offset( dt_local = utc_dt.astimezone(local_tz) - # 'YYYY:MM:DD HH:MM:SS' dt_str = dt_local.strftime("%Y:%m:%d %H:%M:%S") - # '+/-HHMM' -> '+/-HH:MM' (needed format for tags) + # Tags require offset in ±HH:MM format offset_str: str = dt_local.strftime("%z") if len(offset_str) == 5: offset_str = offset_str[:3] + ":" + offset_str[3:] return dt_str, offset_str - -from tqdm.asyncio import tqdm - - +progress_bar = tqdm( + total=df1.shape[0], + desc="Updating EXIF", + unit="file", + disable=False, +) def update_exif_with_exiftool(row: pd.Series) -> None: progress_bar.update(1) @@ -271,7 +254,7 @@ def update_exif_with_exiftool(row: pd.Series) -> None: lat_ref = "N" if latitude >= 0 else "S" lon_ref = "E" if longitude >= 0 else "W" - dt_str, offset_str = get_localized_dt_and_offset( + local_dt, dynamic_tz = get_localized_dt_and_offset( utc_dt=dt_utc, latitude=row["correct_latitude"], longitude=row["correct_longitude"], @@ -283,44 +266,52 @@ def update_exif_with_exiftool(row: pd.Series) -> None: jpg_command = [ "exiftool", - # Local time - f"-IFD0:DateTime={dt_str}", - f"-ExifIFD:DateTimeOriginal={dt_str}", - f"-ExifIFD:DateTimeDigitized={dt_str}", - # Dynamic offset - f"-OffsetTimeOriginal={offset_str}", - # GPS tags + # Date/Time tags + f"-IFD0:DateTime={local_dt}", + f"-ExifIFD:DateTimeOriginal={local_dt}", + f"-ExifIFD:DateTimeDigitized={local_dt}", + f"-OffsetTimeOriginal={dynamic_tz}", + # Location tags f"-GPSLatitude={latitude}", f"-GPSLongitude={longitude}", f"-GPSLatitudeRef='{lat_ref}'", f"-GPSLongitudeRef='{lon_ref}'", - # Keep overwrite and target file + "-overwrite_original", + image_path, + ] + jpg_command_no_gps = [ + "exiftool", + # Date/Time tags + f"-IFD0:DateTime={local_dt}", + f"-ExifIFD:DateTimeOriginal={local_dt}", + f"-ExifIFD:DateTimeDigitized={local_dt}", + f"-OffsetTimeOriginal={dynamic_tz}", "-overwrite_original", image_path, ] mp4_command = [ "exiftool", + # Date/Time tags # EXIF Tags - f"-IFD0:DateTime={dt_str}", - f"-ExifIFD:DateTimeOriginal={dt_str}", - f"-ExifIFD:DateTimeDigitized={dt_str}", - f"-OffsetTimeOriginal={offset_str}", + f"-IFD0:DateTime={local_dt}", + f"-ExifIFD:DateTimeOriginal={local_dt}", + f"-ExifIFD:DateTimeDigitized={local_dt}", + f"-OffsetTimeOriginal={dynamic_tz}", # QuickTime tags (must be set to UTC time) f"-QuickTime:CreationDate={dt_utc_str}", f"-QuickTime:TrackCreateDate={dt_utc_str}", f"-QuickTime:MediaCreateDate={dt_utc_str}", - # This is for the display TimeZone of the above tags - # Not 100% sure needed, doesn't seem to hurt - f"-QuickTime:TimeZone={offset_str}", - # Composite Tag (Needed for location on Google Photos) + f"-QuickTime:TimeZone={dynamic_tz}", + # Location tags + # Needed for location on Google Photos f"-QuickTime:GPSCoordinates={abs_latitude} {lat_ref}, {abs_longitude} {lon_ref}", - # XMP Tags (Adobe/Google compatibility) + # XMP Tags (Adobe/Google f"-xmp:gpslatitude={latitude}", f"-xmp:gpslongitude={longitude}", - # QuickTime Tags (Apple/iOS compatibility) + # QuickTime Tags (Apple/iOS) f"-QuickTime:LocationLatitude={latitude}", f"-QuickTime:LocationLongitude={longitude}", - # # Generic GPS tags (For broad compatibility) + # # Generic GPS tags (Broader compatibility) f"-GPSLatitude={latitude}", f"-GPSLongitude={longitude}", f"-GPSLatitudeRef={lat_ref}", @@ -328,16 +319,41 @@ def update_exif_with_exiftool(row: pd.Series) -> None: "-overwrite_original", image_path, ] + mp4_command_no_gps = [ + "exiftool", + # EXIF Tags + f"-IFD0:DateTime={local_dt}", + f"-ExifIFD:DateTimeOriginal={local_dt}", + f"-ExifIFD:DateTimeDigitized={local_dt}", + f"-OffsetTimeOriginal={dynamic_tz}", + # QuickTime tags (must be set to UTC time) + f"-QuickTime:CreationDate={dt_utc_str}", + f"-QuickTime:TrackCreateDate={dt_utc_str}", + f"-QuickTime:MediaCreateDate={dt_utc_str}", + f"-QuickTime:TimeZone={dynamic_tz}", + "-overwrite_original", + image_path, + ] try: if file_type == "Video": - result = subprocess.run( - mp4_command, check=True, capture_output=True, text=True - ) + if latitude == 0.0 and longitude == 0.0: + result = subprocess.run( + mp4_command_no_gps, check=True, capture_output=True, text=True + ) + else: + result = subprocess.run( + mp4_command, check=True, capture_output=True, text=True + ) elif file_type == "Image": - result = subprocess.run( - jpg_command, check=True, capture_output=True, text=True - ) + if latitude == 0.0 and longitude == 0.0: + result = subprocess.run( + jpg_command_no_gps, check=True, capture_output=True, text=True + ) + else: + result = subprocess.run( + jpg_command, check=True, capture_output=True, text=True + ) else: print(f"⚠️ Warning: Unknown media type for {image_path}, skipping.") return @@ -356,14 +372,6 @@ def update_exif_with_exiftool(row: pd.Series) -> None: ) -progress_bar = tqdm( - total=df1.shape[0], - desc="Updating EXIF", - unit="file", - disable=False, -) - -# img_errors.apply(update_exif_with_exiftool, axis=1) # type: ignore df1.apply(update_exif_with_exiftool, axis=1) # type: ignore - +progress_bar.close()