bad changes
This commit is contained in:
Josh Flatt
parent
af8c3edc96
commit
bd586a9541
@ -1,15 +1,23 @@
|
||||
import subprocess
|
||||
from tqdm.asyncio import tqdm
|
||||
from pathlib import Path
|
||||
import pandas as pd
|
||||
|
||||
|
||||
file_directory = Path("../downloads")
|
||||
file_directory = Path("./downloads")
|
||||
files_to_check = [
|
||||
p
|
||||
for p in file_directory.iterdir()
|
||||
if p.is_file() and p.suffix.lower() in [".jpg", ".mp4"]
|
||||
]
|
||||
|
||||
progress_bar = tqdm(
|
||||
total=len(files_to_check),
|
||||
desc="Gathering",
|
||||
unit="file",
|
||||
disable=False,
|
||||
)
|
||||
|
||||
file_df = pd.DataFrame(data=files_to_check, columns=["file"])
|
||||
|
||||
|
||||
@ -32,13 +40,17 @@ def get_metadata(row: pd.Series) -> pd.Series:
|
||||
|
||||
if len(metadata) < 3:
|
||||
metadata = metadata + ["-"] * (3 - len(metadata))
|
||||
# row["file"] = f"../downloads/" + str(Path.resolve(row["file"]))
|
||||
row["createdate"] = metadata[0]
|
||||
row["gpslatitude"] = metadata[1]
|
||||
row["gpslongitude"] = metadata[2]
|
||||
progress_bar.update(1)
|
||||
return row
|
||||
|
||||
|
||||
df = file_df.apply(get_metadata, axis=1)
|
||||
|
||||
# df["file"] = f"../downloads/" + str(Path.resolve(df["file"])))
|
||||
|
||||
df.to_csv("../temp/filemetadata.csv", index=False, sep="#")
|
||||
|
||||
df.to_csv("./temp/filemetadata.csv", index=False, sep="#")
|
||||
|
||||
8
main.py
8
main.py
@ -165,11 +165,11 @@ async def download_memory(
|
||||
timestamp = memory.date.timestamp()
|
||||
os.utime(output_path, (timestamp, timestamp))
|
||||
|
||||
if add_exif and ext == ".jpg":
|
||||
add_exif_data(output_path, memory)
|
||||
# if add_exif and ext == ".jpg":
|
||||
# add_exif_data(output_path, memory)
|
||||
|
||||
elif add_exif and ext == ".mp4":
|
||||
add_metadata_mp4(output_path, memory)
|
||||
# elif add_exif and ext == ".mp4":
|
||||
# add_metadata_mp4(output_path, memory)
|
||||
|
||||
return True, len(response.content)
|
||||
except Exception as e:
|
||||
|
||||
139
main3.py
Normal file
139
main3.py
Normal file
@ -0,0 +1,139 @@
|
||||
import asyncio
|
||||
import aiohttp
|
||||
import json
|
||||
import pytz
|
||||
import time
|
||||
import os
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from tqdm import tqdm
|
||||
|
||||
CONCURRENCY = 50
|
||||
RETRIES = 3
|
||||
OUTPUT_DIR = Path("./downloads")
|
||||
CHECKPOINT = Path("./temp/checkpoint.txt")
|
||||
|
||||
|
||||
def load_checkpoint():
|
||||
if CHECKPOINT.exists():
|
||||
return set(CHECKPOINT.read_text().splitlines())
|
||||
return set()
|
||||
|
||||
|
||||
def save_to_checkpoint(path: Path):
|
||||
with CHECKPOINT.open("a") as f:
|
||||
f.write(str(path) + "\n")
|
||||
|
||||
|
||||
# def utc_filename(timestamp: str, url: str) -> Path:
|
||||
# dt = datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
|
||||
# ext = "mp4" if ".mp4" in url.lower() else "jpg"
|
||||
# return OUTPUT_DIR / f"{dt.strftime('%Y-%m-%d_%H-%M-%S')}.{ext}"
|
||||
|
||||
def utc_filename(timestamp: str, url: str) -> Path:
|
||||
# parse "YYYY-MM-DD HH:MM:SS UTC"
|
||||
dt = datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S UTC")
|
||||
dt = dt.replace(tzinfo=pytz.utc)
|
||||
ext = "mp4" if ".mp4" in url.lower() else "jpg"
|
||||
out_path = os.path.join("./downloads",f"{dt.strftime('%Y-%m-%d_%H-%M-%S')}.{ext}")
|
||||
print(out_path)
|
||||
return Path(out_path)
|
||||
|
||||
|
||||
async def download_one(session, sem, url, path, failures, stats):
|
||||
if path.exists():
|
||||
return
|
||||
|
||||
# Already downloaded? (checkpoint resume)
|
||||
if str(path) in stats["done"]:
|
||||
return
|
||||
|
||||
async with sem:
|
||||
for attempt in range(1, RETRIES + 1):
|
||||
try:
|
||||
async with session.get(url) as resp:
|
||||
resp.raise_for_status()
|
||||
data = await resp.read()
|
||||
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(data)
|
||||
|
||||
# record MB
|
||||
stats["mb"] += len(data) / (1024 * 1024)
|
||||
|
||||
# checkpoint
|
||||
save_to_checkpoint(path)
|
||||
stats["done"].add(str(path))
|
||||
|
||||
return
|
||||
except Exception as e:
|
||||
if attempt == RETRIES:
|
||||
failures.append((url, str(e)))
|
||||
await asyncio.sleep(0.3 * attempt) # small backoff
|
||||
|
||||
|
||||
async def main():
|
||||
# Load JSON
|
||||
with open("./resources/json/memories_history.json", "r") as f:
|
||||
memories = json.load(f)["Saved Media"]
|
||||
|
||||
# Prepare tasks
|
||||
tasks_to_download = []
|
||||
for item in memories:
|
||||
url = item.get("Download Link")
|
||||
ts = item.get("Date")
|
||||
if not url or not ts:
|
||||
continue
|
||||
outpath = utc_filename(ts, url)
|
||||
tasks_to_download.append((url, outpath))
|
||||
|
||||
stats = {
|
||||
"mb": 0.0,
|
||||
"done": load_checkpoint() # resume if exists
|
||||
}
|
||||
|
||||
failures = []
|
||||
sem = asyncio.Semaphore(CONCURRENCY)
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
start_time = time.time()
|
||||
|
||||
# tqdm based on number of files not yet done
|
||||
to_process = [
|
||||
(url, path) for (url, path) in tasks_to_download
|
||||
if not path.exists() and str(path) not in stats["done"]
|
||||
]
|
||||
|
||||
progress = tqdm(total=len(to_process), desc="Downloading", unit="file")
|
||||
|
||||
# Wrap each task so progress bar updates *after* the download completes
|
||||
async def wrapped(url, path):
|
||||
await download_one(session, sem, url, path, failures, stats)
|
||||
progress.update(1)
|
||||
|
||||
# Start tasks in batches
|
||||
await asyncio.gather(*(wrapped(url, path) for url, path in to_process))
|
||||
progress.close()
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
|
||||
# Final summary
|
||||
mb_total = stats["mb"]
|
||||
speed = mb_total / elapsed if elapsed > 0 else 0
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print(f"Downloaded: {len(to_process) - len(failures)} files")
|
||||
print(f"Failed: {len(failures)} files")
|
||||
print(f"Data: {mb_total:.2f} MB")
|
||||
print(f"Speed: {speed:.2f} MB/s")
|
||||
print("=" * 60)
|
||||
|
||||
if failures:
|
||||
print("\n⚠️ Failed downloads:")
|
||||
for url, err in failures:
|
||||
print(f" - {url} ({err})")
|
||||
print()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
File diff suppressed because it is too large
Load Diff
@ -65,43 +65,43 @@
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>0</th>\n",
|
||||
" <td>../downloads/2019-04-05_23-27-17.jpg</td>\n",
|
||||
" <td>../downloads/2025-03-15_01-17-10.mp4</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1</th>\n",
|
||||
" <td>../downloads/2024-10-23_00-29-09.jpg</td>\n",
|
||||
" <td>../downloads/2025-02-28_18-25-46.mp4</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2</th>\n",
|
||||
" <td>../downloads/2025-02-16_00-09-41.mp4</td>\n",
|
||||
" <td>../downloads/2024-09-09_23-21-56.jpg</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>3</th>\n",
|
||||
" <td>../downloads/2023-03-10_22-11-18.jpg</td>\n",
|
||||
" <td>../downloads/2025-02-18_13-40-28.mp4</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>4</th>\n",
|
||||
" <td>../downloads/2024-02-24_22-18-07.jpg</td>\n",
|
||||
" <td>../downloads/2024-10-18_02-04-34.mp4</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>5</th>\n",
|
||||
" <td>../downloads/2020-10-24_03-27-27.mp4</td>\n",
|
||||
" <td>../downloads/2025-03-08_06-20-39.mp4</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>6</th>\n",
|
||||
" <td>../downloads/2020-11-04_04-40-23.mp4</td>\n",
|
||||
" <td>../downloads/2025-06-28_02-54-42.jpg</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>7</th>\n",
|
||||
" <td>../downloads/2017-06-09_22-44-18.jpg</td>\n",
|
||||
" <td>../downloads/2025-04-13_22-54-57.jpg</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>8</th>\n",
|
||||
" <td>../downloads/2018-12-26_06-49-23.mp4</td>\n",
|
||||
" <td>../downloads/2025-03-04_19-48-15.mp4</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>9</th>\n",
|
||||
" <td>../downloads/2024-07-25_04-57-56.mp4</td>\n",
|
||||
" <td>../downloads/2025-04-28_21-29-43.mp4</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
@ -109,16 +109,16 @@
|
||||
],
|
||||
"text/plain": [
|
||||
" file\n",
|
||||
"0 ../downloads/2019-04-05_23-27-17.jpg\n",
|
||||
"1 ../downloads/2024-10-23_00-29-09.jpg\n",
|
||||
"2 ../downloads/2025-02-16_00-09-41.mp4\n",
|
||||
"3 ../downloads/2023-03-10_22-11-18.jpg\n",
|
||||
"4 ../downloads/2024-02-24_22-18-07.jpg\n",
|
||||
"5 ../downloads/2020-10-24_03-27-27.mp4\n",
|
||||
"6 ../downloads/2020-11-04_04-40-23.mp4\n",
|
||||
"7 ../downloads/2017-06-09_22-44-18.jpg\n",
|
||||
"8 ../downloads/2018-12-26_06-49-23.mp4\n",
|
||||
"9 ../downloads/2024-07-25_04-57-56.mp4"
|
||||
"0 ../downloads/2025-03-15_01-17-10.mp4\n",
|
||||
"1 ../downloads/2025-02-28_18-25-46.mp4\n",
|
||||
"2 ../downloads/2024-09-09_23-21-56.jpg\n",
|
||||
"3 ../downloads/2025-02-18_13-40-28.mp4\n",
|
||||
"4 ../downloads/2024-10-18_02-04-34.mp4\n",
|
||||
"5 ../downloads/2025-03-08_06-20-39.mp4\n",
|
||||
"6 ../downloads/2025-06-28_02-54-42.jpg\n",
|
||||
"7 ../downloads/2025-04-13_22-54-57.jpg\n",
|
||||
"8 ../downloads/2025-03-04_19-48-15.mp4\n",
|
||||
"9 ../downloads/2025-04-28_21-29-43.mp4"
|
||||
]
|
||||
},
|
||||
"execution_count": 3,
|
||||
@ -175,7 +175,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"execution_count": 6,
|
||||
"id": "e4389a3e",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
@ -209,105 +209,105 @@
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>0</th>\n",
|
||||
" <td>../downloads/2019-04-05_23-27-17.jpg</td>\n",
|
||||
" <td>2019:04:05 18:27:17</td>\n",
|
||||
" <td>40 deg 24' 14.76\" N</td>\n",
|
||||
" <td>104 deg 44' 21.52\" W</td>\n",
|
||||
" <td>../downloads/2025-03-15_01-17-10.mp4</td>\n",
|
||||
" <td>2025:03:15 01:17:29</td>\n",
|
||||
" <td>-</td>\n",
|
||||
" <td>-</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1</th>\n",
|
||||
" <td>../downloads/2024-10-23_00-29-09.jpg</td>\n",
|
||||
" <td>2024:10:22 19:29:09</td>\n",
|
||||
" <td>44 deg 6' 15.77\" N</td>\n",
|
||||
" <td>94 deg 13' 10.52\" W</td>\n",
|
||||
" <td>../downloads/2025-02-28_18-25-46.mp4</td>\n",
|
||||
" <td>2025:02:28 18:26:16</td>\n",
|
||||
" <td>-</td>\n",
|
||||
" <td>-</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2</th>\n",
|
||||
" <td>../downloads/2025-02-16_00-09-41.mp4</td>\n",
|
||||
" <td>2025:02:15 18:09:41</td>\n",
|
||||
" <td>44 deg 21' 17.71\" N</td>\n",
|
||||
" <td>93 deg 58' 1.20\" W</td>\n",
|
||||
" <td>../downloads/2024-09-09_23-21-56.jpg</td>\n",
|
||||
" <td>-</td>\n",
|
||||
" <td>-</td>\n",
|
||||
" <td>-</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>3</th>\n",
|
||||
" <td>../downloads/2023-03-10_22-11-18.jpg</td>\n",
|
||||
" <td>2023:03:10 16:11:18</td>\n",
|
||||
" <td>40 deg 24' 39.13\" N</td>\n",
|
||||
" <td>104 deg 41' 33.90\" W</td>\n",
|
||||
" <td>../downloads/2025-02-18_13-40-28.mp4</td>\n",
|
||||
" <td>2025:02:18 13:41:28</td>\n",
|
||||
" <td>-</td>\n",
|
||||
" <td>-</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>4</th>\n",
|
||||
" <td>../downloads/2024-02-24_22-18-07.jpg</td>\n",
|
||||
" <td>2024:02:24 16:18:07</td>\n",
|
||||
" <td>40 deg 25' 17.08\" N</td>\n",
|
||||
" <td>104 deg 45' 22.75\" W</td>\n",
|
||||
" <td>../downloads/2024-10-18_02-04-34.mp4</td>\n",
|
||||
" <td>2024:10:18 02:04:52</td>\n",
|
||||
" <td>-</td>\n",
|
||||
" <td>-</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>5</th>\n",
|
||||
" <td>../downloads/2020-10-24_03-27-27.mp4</td>\n",
|
||||
" <td>2020:10:23 21:27:27</td>\n",
|
||||
" <td>40 deg 24' 37.62\" N</td>\n",
|
||||
" <td>104 deg 42' 28.73\" W</td>\n",
|
||||
" <td>../downloads/2025-03-08_06-20-39.mp4</td>\n",
|
||||
" <td>2025:03:08 06:21:26</td>\n",
|
||||
" <td>-</td>\n",
|
||||
" <td>-</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>6</th>\n",
|
||||
" <td>../downloads/2020-11-04_04-40-23.mp4</td>\n",
|
||||
" <td>2020:11:03 21:40:23</td>\n",
|
||||
" <td>40 deg 25' 55.63\" N</td>\n",
|
||||
" <td>104 deg 46' 10.16\" W</td>\n",
|
||||
" <td>../downloads/2025-06-28_02-54-42.jpg</td>\n",
|
||||
" <td>-</td>\n",
|
||||
" <td>-</td>\n",
|
||||
" <td>-</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>7</th>\n",
|
||||
" <td>../downloads/2017-06-09_22-44-18.jpg</td>\n",
|
||||
" <td>2017:06:09 17:44:18</td>\n",
|
||||
" <td>39 deg 50' 49.74\" N</td>\n",
|
||||
" <td>104 deg 40' 29.57\" W</td>\n",
|
||||
" <td>../downloads/2025-04-13_22-54-57.jpg</td>\n",
|
||||
" <td>-</td>\n",
|
||||
" <td>-</td>\n",
|
||||
" <td>-</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>8</th>\n",
|
||||
" <td>../downloads/2018-12-26_06-49-23.mp4</td>\n",
|
||||
" <td>2018:12:25 23:49:23</td>\n",
|
||||
" <td>40 deg 25' 56.21\" N</td>\n",
|
||||
" <td>104 deg 46' 11.24\" W</td>\n",
|
||||
" <td>../downloads/2025-03-04_19-48-15.mp4</td>\n",
|
||||
" <td>2025:03:04 19:48:37</td>\n",
|
||||
" <td>-</td>\n",
|
||||
" <td>-</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>9</th>\n",
|
||||
" <td>../downloads/2024-07-25_04-57-56.mp4</td>\n",
|
||||
" <td>2024:07:24 23:57:56</td>\n",
|
||||
" <td>44 deg 21' 17.17\" N</td>\n",
|
||||
" <td>93 deg 57' 57.82\" W</td>\n",
|
||||
" <td>../downloads/2025-04-28_21-29-43.mp4</td>\n",
|
||||
" <td>2025:04:28 21:30:00</td>\n",
|
||||
" <td>-</td>\n",
|
||||
" <td>-</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" file createdate \\\n",
|
||||
"0 ../downloads/2019-04-05_23-27-17.jpg 2019:04:05 18:27:17 \n",
|
||||
"1 ../downloads/2024-10-23_00-29-09.jpg 2024:10:22 19:29:09 \n",
|
||||
"2 ../downloads/2025-02-16_00-09-41.mp4 2025:02:15 18:09:41 \n",
|
||||
"3 ../downloads/2023-03-10_22-11-18.jpg 2023:03:10 16:11:18 \n",
|
||||
"4 ../downloads/2024-02-24_22-18-07.jpg 2024:02:24 16:18:07 \n",
|
||||
"5 ../downloads/2020-10-24_03-27-27.mp4 2020:10:23 21:27:27 \n",
|
||||
"6 ../downloads/2020-11-04_04-40-23.mp4 2020:11:03 21:40:23 \n",
|
||||
"7 ../downloads/2017-06-09_22-44-18.jpg 2017:06:09 17:44:18 \n",
|
||||
"8 ../downloads/2018-12-26_06-49-23.mp4 2018:12:25 23:49:23 \n",
|
||||
"9 ../downloads/2024-07-25_04-57-56.mp4 2024:07:24 23:57:56 \n",
|
||||
" file createdate gpslatitude \\\n",
|
||||
"0 ../downloads/2025-03-15_01-17-10.mp4 2025:03:15 01:17:29 - \n",
|
||||
"1 ../downloads/2025-02-28_18-25-46.mp4 2025:02:28 18:26:16 - \n",
|
||||
"2 ../downloads/2024-09-09_23-21-56.jpg - - \n",
|
||||
"3 ../downloads/2025-02-18_13-40-28.mp4 2025:02:18 13:41:28 - \n",
|
||||
"4 ../downloads/2024-10-18_02-04-34.mp4 2024:10:18 02:04:52 - \n",
|
||||
"5 ../downloads/2025-03-08_06-20-39.mp4 2025:03:08 06:21:26 - \n",
|
||||
"6 ../downloads/2025-06-28_02-54-42.jpg - - \n",
|
||||
"7 ../downloads/2025-04-13_22-54-57.jpg - - \n",
|
||||
"8 ../downloads/2025-03-04_19-48-15.mp4 2025:03:04 19:48:37 - \n",
|
||||
"9 ../downloads/2025-04-28_21-29-43.mp4 2025:04:28 21:30:00 - \n",
|
||||
"\n",
|
||||
" gpslatitude gpslongitude \n",
|
||||
"0 40 deg 24' 14.76\" N 104 deg 44' 21.52\" W \n",
|
||||
"1 44 deg 6' 15.77\" N 94 deg 13' 10.52\" W \n",
|
||||
"2 44 deg 21' 17.71\" N 93 deg 58' 1.20\" W \n",
|
||||
"3 40 deg 24' 39.13\" N 104 deg 41' 33.90\" W \n",
|
||||
"4 40 deg 25' 17.08\" N 104 deg 45' 22.75\" W \n",
|
||||
"5 40 deg 24' 37.62\" N 104 deg 42' 28.73\" W \n",
|
||||
"6 40 deg 25' 55.63\" N 104 deg 46' 10.16\" W \n",
|
||||
"7 39 deg 50' 49.74\" N 104 deg 40' 29.57\" W \n",
|
||||
"8 40 deg 25' 56.21\" N 104 deg 46' 11.24\" W \n",
|
||||
"9 44 deg 21' 17.17\" N 93 deg 57' 57.82\" W "
|
||||
" gpslongitude \n",
|
||||
"0 - \n",
|
||||
"1 - \n",
|
||||
"2 - \n",
|
||||
"3 - \n",
|
||||
"4 - \n",
|
||||
"5 - \n",
|
||||
"6 - \n",
|
||||
"7 - \n",
|
||||
"8 - \n",
|
||||
"9 - "
|
||||
]
|
||||
},
|
||||
"execution_count": 8,
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
@ -343,7 +343,7 @@
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.13.11"
|
||||
"version": "3.12.6"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
|
||||
@ -5,6 +5,7 @@ tqdm>=4.67.1
|
||||
pillow
|
||||
piexif
|
||||
timezonefinder
|
||||
aiohttp
|
||||
|
||||
# For development
|
||||
jupyter
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
3711
temp/filemetadata0.csv
Normal file
3711
temp/filemetadata0.csv
Normal file
File diff suppressed because it is too large
Load Diff
369
update_files.py
Normal file
369
update_files.py
Normal file
@ -0,0 +1,369 @@
|
||||
|
||||
import pandas as pd
|
||||
import re
|
||||
import pytz
|
||||
import os
|
||||
from timezonefinder import TimezoneFinder
|
||||
from datetime import datetime
|
||||
import subprocess
|
||||
|
||||
|
||||
df: pd.DataFrame = pd.read_json("./temp/file_info.json", orient="records")
|
||||
df.head(5)
|
||||
|
||||
|
||||
def get_join_col(row: pd.Series) -> pd.Series:
|
||||
row["file_dt"] = f"{row['Date'].strftime('%Y-%m-%d_%H-%M-%S')}"
|
||||
return row
|
||||
|
||||
|
||||
def extract_lat_long(row: pd.Series) -> pd.Series:
|
||||
coordinates = row["Location"].split(": ")[1]
|
||||
lat_str, long_str = coordinates.split(", ")
|
||||
row["location_latitude"] = round(float(lat_str), 5)
|
||||
row["location_longitude"] = round(float(long_str), 5)
|
||||
return row
|
||||
|
||||
|
||||
df = df.apply(get_join_col, axis=1)
|
||||
df = df.apply(extract_lat_long, axis=1)
|
||||
df.head(5)
|
||||
|
||||
|
||||
metadata = pd.read_csv("./temp/filemetadata.csv", sep="#")
|
||||
metadata["file"] = f"./" + metadata["file"]
|
||||
metadata.head(5)
|
||||
|
||||
|
||||
def convert_lat_long_to_decimal_degrees(row: pd.Series) -> pd.Series:
|
||||
def dms_to_decimal(dms_string):
|
||||
match = re.search(
|
||||
r"(\d+)\s*deg\s*(\d+)'\s*([\d.]+)\"\s*([NSEW])", dms_string, re.IGNORECASE
|
||||
)
|
||||
if not match:
|
||||
raise ValueError(f"Coordinate format not recognized: {dms_string}")
|
||||
|
||||
degrees = float(match.group(1))
|
||||
minutes = float(match.group(2))
|
||||
seconds = float(match.group(3))
|
||||
direction = match.group(4).upper()
|
||||
decimal_degrees = degrees + (minutes / 60) + (seconds / 3600)
|
||||
if direction in ["S", "W"]:
|
||||
decimal_degrees *= -1
|
||||
decimal_degrees = round(decimal_degrees, 5)
|
||||
return decimal_degrees
|
||||
|
||||
def convert(value: str) -> float:
|
||||
if pd.isna(value):
|
||||
return 0.0
|
||||
if value == "-":
|
||||
return 0.0
|
||||
try:
|
||||
return dms_to_decimal(value)
|
||||
except ValueError:
|
||||
return 0.0
|
||||
|
||||
row["Latitude"] = convert(row["gpslatitude"])
|
||||
row["Longitude"] = convert(row["gpslongitude"])
|
||||
return row
|
||||
|
||||
|
||||
def get_file_name(row: pd.Series) -> pd.Series:
|
||||
path_parts = row["file"].split("/")
|
||||
file = path_parts[-1]
|
||||
row["filename"] = file
|
||||
row["extension"] = file.split(".")[-1].lower()
|
||||
row["file_dt"] = file.split(".")[0]
|
||||
return row
|
||||
|
||||
|
||||
meta = metadata.apply(convert_lat_long_to_decimal_degrees, axis=1)
|
||||
meta = meta.apply(get_file_name, axis=1)
|
||||
meta.head(5)
|
||||
|
||||
|
||||
def get_datetime_for_df(row: pd.Series) -> pd.Series:
|
||||
from datetime import datetime
|
||||
|
||||
try:
|
||||
row["createdate_mod"] = None
|
||||
if row["createdate"] != "-":
|
||||
createdate = datetime.strptime(row["createdate"], "%Y:%m:%d %H:%M:%S")
|
||||
createdate_wtz = createdate.astimezone(pytz.timezone("America/Chicago"))
|
||||
row["createdate_mod"] = createdate_wtz
|
||||
|
||||
date = datetime.strptime(row["file_dt"], "%Y-%m-%d_%H-%M-%S")
|
||||
dt_w_tz = date.astimezone(pytz.UTC)
|
||||
row["datetime"] = dt_w_tz
|
||||
row["dt_join"] = f"{dt_w_tz}"
|
||||
except ValueError:
|
||||
row["createdate_mod"] = None
|
||||
row["datetime"] = None
|
||||
row["dt_join"] = ""
|
||||
return row
|
||||
|
||||
|
||||
def get_media_type(row: pd.Series) -> pd.Series:
|
||||
row["file_type"] = "Unknown"
|
||||
if row["extension"] == "jpg":
|
||||
row["file_type"] = "Image"
|
||||
if row["extension"] == "mp4":
|
||||
row["file_type"] = "Video"
|
||||
return row
|
||||
|
||||
|
||||
data = meta.apply(get_datetime_for_df, axis=1)
|
||||
data = data.apply(get_media_type, axis=1)
|
||||
joined = pd.merge(
|
||||
df,
|
||||
data,
|
||||
how="left",
|
||||
left_on="file_dt",
|
||||
right_on="file_dt",
|
||||
suffixes=("_df", "_meta"),
|
||||
)
|
||||
|
||||
|
||||
col_mapper = {
|
||||
"Date": "correct_date_utc",
|
||||
"Media Type": "correct_media_type",
|
||||
"location_latitude": "correct_latitude",
|
||||
"location_longitude": "correct_longitude",
|
||||
"file": "path",
|
||||
"filename": "filename",
|
||||
"createdate": "actual_date_cst",
|
||||
"Latitude": "actual_latitude",
|
||||
"Longitude": "actual_longitude",
|
||||
"file_type": "actual_media_type",
|
||||
}
|
||||
df0 = joined[col_mapper.keys()].rename(columns=col_mapper)
|
||||
|
||||
|
||||
def check_for_errors(row: pd.Series) -> pd.Series:
|
||||
errors = 0
|
||||
if row["correct_media_type"] != row["actual_media_type"]:
|
||||
errors += 1
|
||||
if row["actual_latitude"] == 0.00 and row["correct_media_type"] == "Image":
|
||||
if row["correct_latitude"] != 0.00:
|
||||
errors += 1
|
||||
if abs(row["actual_longitude"] - row["correct_longitude"]) > 2.00:
|
||||
errors += 1
|
||||
if row["actual_date_cst"] == "-":
|
||||
errors += 1
|
||||
if pd.isna(row["path"]):
|
||||
errors += 1
|
||||
row["errors"] = errors
|
||||
return row
|
||||
|
||||
|
||||
df1 = df0.apply(check_for_errors, axis=1)
|
||||
|
||||
|
||||
errors = df1[df1["errors"] > 0]
|
||||
errors.head(5)
|
||||
|
||||
|
||||
print(f"Errors: {errors.shape[0]}")
|
||||
|
||||
|
||||
img_errors = errors[errors["correct_media_type"] == "Image"]
|
||||
print(f"Image errors: {img_errors.shape[0]}")
|
||||
|
||||
|
||||
video_errors = errors[errors["correct_media_type"] == "Video"]
|
||||
print(f"Video errors: {video_errors.shape[0]}")
|
||||
|
||||
|
||||
video_errors.head(5)
|
||||
|
||||
|
||||
def fix_filetype(row: pd.Series) -> pd.Series:
|
||||
try:
|
||||
if pd.isna(row["path"]) or row["path"] == "-":
|
||||
return row
|
||||
|
||||
mp4_file_path = f"./downloads/{row['filename'][:-4]}.mp4"
|
||||
jpg_file_path = f"./downloads/{row['filename'][:-4]}.jpg"
|
||||
|
||||
if row["correct_media_type"] == row["actual_media_type"]:
|
||||
return row
|
||||
|
||||
if row["path"] == "-":
|
||||
return row
|
||||
|
||||
if row["correct_media_type"] == "Image":
|
||||
os.rename(src=mp4_file_path, dst=jpg_file_path)
|
||||
if os.path.exists(jpg_file_path):
|
||||
row["filename"] = f"{row['filename'][:-4]}.jpg"
|
||||
row["actual_media_type"] = "Image"
|
||||
elif row["correct_media_type"] == "Video":
|
||||
os.rename(src=jpg_file_path, dst=mp4_file_path)
|
||||
if os.path.exists(mp4_file_path):
|
||||
row["filename"] = f"{row['filename'][:-4]}.mp4"
|
||||
row["actual_media_type"] = "Video"
|
||||
except FileNotFoundError:
|
||||
print(f"File not found: {row['filename']}")
|
||||
return row
|
||||
|
||||
|
||||
# wrong_filetype = df1[df1["correct_media_type"] != df1["actual_media_type"]]
|
||||
# wrong_filetype = wrong_filetype.fillna("-")
|
||||
# wrong_filetype.head(5)
|
||||
|
||||
|
||||
# wrong_filetype.apply(fix_filetype, axis=1)
|
||||
df1.apply(fix_filetype, axis=1)
|
||||
|
||||
|
||||
tf = TimezoneFinder()
|
||||
|
||||
|
||||
def get_localized_dt_and_offset(
|
||||
utc_dt: datetime, latitude: float, longitude: float
|
||||
) -> tuple[str, str]:
|
||||
tz_name = tf.timezone_at(lng=longitude, lat=latitude)
|
||||
|
||||
utc_timezone = pytz.utc
|
||||
central_timezone = pytz.timezone("America/Chicago")
|
||||
mountain_timezone = pytz.timezone("America/Denver")
|
||||
|
||||
if not tz_name or latitude == 0 or longitude == 0:
|
||||
print(
|
||||
f"⚠️ Warning: No timezone found for {latitude}, {longitude}. Defaulting to MDT/MST."
|
||||
)
|
||||
local_tz = mountain_timezone
|
||||
else:
|
||||
local_tz = pytz.timezone(tz_name)
|
||||
|
||||
dt_local = utc_dt.astimezone(local_tz)
|
||||
|
||||
# 'YYYY:MM:DD HH:MM:SS'
|
||||
dt_str = dt_local.strftime("%Y:%m:%d %H:%M:%S")
|
||||
|
||||
# '+/-HHMM' -> '+/-HH:MM' (needed format for tags)
|
||||
offset_str: str = dt_local.strftime("%z")
|
||||
if len(offset_str) == 5:
|
||||
offset_str = offset_str[:3] + ":" + offset_str[3:]
|
||||
|
||||
return dt_str, offset_str
|
||||
|
||||
|
||||
from tqdm.asyncio import tqdm
|
||||
|
||||
|
||||
|
||||
def update_exif_with_exiftool(row: pd.Series) -> None:
|
||||
progress_bar.update(1)
|
||||
image_path = row["path"]
|
||||
dt_utc = row["correct_date_utc"] # timezone-aware, UTC datetime
|
||||
latitude = row["correct_latitude"]
|
||||
longitude = row["correct_longitude"]
|
||||
file_type = row["correct_media_type"]
|
||||
abs_latitude = abs(latitude)
|
||||
abs_longitude = abs(longitude)
|
||||
|
||||
dt_utc_str = dt_utc.strftime("%Y:%m:%d %H:%M:%S")
|
||||
|
||||
if image_path is None or pd.isna(image_path) or image_path == "":
|
||||
print(f"❌ File not found, skipping: {image_path}")
|
||||
return
|
||||
|
||||
lat_ref = "N" if latitude >= 0 else "S"
|
||||
lon_ref = "E" if longitude >= 0 else "W"
|
||||
|
||||
dt_str, offset_str = get_localized_dt_and_offset(
|
||||
utc_dt=dt_utc,
|
||||
latitude=row["correct_latitude"],
|
||||
longitude=row["correct_longitude"],
|
||||
)
|
||||
|
||||
if not os.path.exists(image_path):
|
||||
print(f"❌ File not found, skipping: {image_path}")
|
||||
return
|
||||
|
||||
jpg_command = [
|
||||
"exiftool",
|
||||
# Local time
|
||||
f"-IFD0:DateTime={dt_str}",
|
||||
f"-ExifIFD:DateTimeOriginal={dt_str}",
|
||||
f"-ExifIFD:DateTimeDigitized={dt_str}",
|
||||
# Dynamic offset
|
||||
f"-OffsetTimeOriginal={offset_str}",
|
||||
# GPS tags
|
||||
f"-GPSLatitude={latitude}",
|
||||
f"-GPSLongitude={longitude}",
|
||||
f"-GPSLatitudeRef='{lat_ref}'",
|
||||
f"-GPSLongitudeRef='{lon_ref}'",
|
||||
# Keep overwrite and target file
|
||||
"-overwrite_original",
|
||||
image_path,
|
||||
]
|
||||
mp4_command = [
|
||||
"exiftool",
|
||||
# EXIF Tags
|
||||
f"-IFD0:DateTime={dt_str}",
|
||||
f"-ExifIFD:DateTimeOriginal={dt_str}",
|
||||
f"-ExifIFD:DateTimeDigitized={dt_str}",
|
||||
f"-OffsetTimeOriginal={offset_str}",
|
||||
# QuickTime tags (must be set to UTC time)
|
||||
f"-QuickTime:CreationDate={dt_utc_str}",
|
||||
f"-QuickTime:TrackCreateDate={dt_utc_str}",
|
||||
f"-QuickTime:MediaCreateDate={dt_utc_str}",
|
||||
# This is for the display TimeZone of the above tags
|
||||
# Not 100% sure needed, doesn't seem to hurt
|
||||
f"-QuickTime:TimeZone={offset_str}",
|
||||
# Composite Tag (Needed for location on Google Photos)
|
||||
f"-QuickTime:GPSCoordinates={abs_latitude} {lat_ref}, {abs_longitude} {lon_ref}",
|
||||
# XMP Tags (Adobe/Google compatibility)
|
||||
f"-xmp:gpslatitude={latitude}",
|
||||
f"-xmp:gpslongitude={longitude}",
|
||||
# QuickTime Tags (Apple/iOS compatibility)
|
||||
f"-QuickTime:LocationLatitude={latitude}",
|
||||
f"-QuickTime:LocationLongitude={longitude}",
|
||||
# # Generic GPS tags (For broad compatibility)
|
||||
f"-GPSLatitude={latitude}",
|
||||
f"-GPSLongitude={longitude}",
|
||||
f"-GPSLatitudeRef={lat_ref}",
|
||||
f"-GPSLongitudeRef={lon_ref}",
|
||||
"-overwrite_original",
|
||||
image_path,
|
||||
]
|
||||
|
||||
try:
|
||||
if file_type == "Video":
|
||||
result = subprocess.run(
|
||||
mp4_command, check=True, capture_output=True, text=True
|
||||
)
|
||||
elif file_type == "Image":
|
||||
result = subprocess.run(
|
||||
jpg_command, check=True, capture_output=True, text=True
|
||||
)
|
||||
else:
|
||||
print(f"⚠️ Warning: Unknown media type for {image_path}, skipping.")
|
||||
return
|
||||
|
||||
if "1 image files updated" in result.stdout:
|
||||
pass
|
||||
# print(f"✅ Updated: {image_path}")
|
||||
else:
|
||||
print(f"⚠️ Warning updating {image_path}: {result.stderr.strip()}")
|
||||
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"🛑 Subprocess Error on {image_path}: {e.stderr.strip()}")
|
||||
except FileNotFoundError:
|
||||
print(
|
||||
"🛑 ERROR: ExifTool command not found. Is ExifTool installed and in your PATH?"
|
||||
)
|
||||
|
||||
|
||||
progress_bar = tqdm(
|
||||
total=df1.shape[0],
|
||||
desc="Updating EXIF",
|
||||
unit="file",
|
||||
disable=False,
|
||||
)
|
||||
|
||||
# img_errors.apply(update_exif_with_exiftool, axis=1) # type: ignore
|
||||
df1.apply(update_exif_with_exiftool, axis=1) # type: ignore
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user