cleaned up notebook
This commit is contained in:
parent
7e2f426ddd
commit
f624ce357c
@ -2,221 +2,42 @@
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"execution_count": null,
|
||||
"id": "ffcab11d",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import pandas as pd"
|
||||
"import pandas as pd\n",
|
||||
"import re\n",
|
||||
"import pytz\n",
|
||||
"import os\n",
|
||||
"from timezonefinder import TimezoneFinder\n",
|
||||
"from datetime import datetime\n",
|
||||
"import subprocess"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"execution_count": null,
|
||||
"id": "a4cda272",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"df: pd.DataFrame = pd.read_json(\"../temp/file_info.json\", orient='records')"
|
||||
"df: pd.DataFrame = pd.read_json(\"../temp/file_info.json\", orient='records')\n",
|
||||
"df.head(5)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "09ee7182",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div>\n",
|
||||
"<style scoped>\n",
|
||||
" .dataframe tbody tr th:only-of-type {\n",
|
||||
" vertical-align: middle;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe tbody tr th {\n",
|
||||
" vertical-align: top;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe thead th {\n",
|
||||
" text-align: right;\n",
|
||||
" }\n",
|
||||
"</style>\n",
|
||||
"<table border=\"1\" class=\"dataframe\">\n",
|
||||
" <thead>\n",
|
||||
" <tr style=\"text-align: right;\">\n",
|
||||
" <th></th>\n",
|
||||
" <th>Date</th>\n",
|
||||
" <th>Media Type</th>\n",
|
||||
" <th>Location</th>\n",
|
||||
" <th>Download Link</th>\n",
|
||||
" <th>Media Download Url</th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>0</th>\n",
|
||||
" <td>2025-11-29 04:50:47+00:00</td>\n",
|
||||
" <td>Video</td>\n",
|
||||
" <td>Latitude, Longitude: 44.110836, -94.217804</td>\n",
|
||||
" <td>https://app.snapchat.com/dmd/memories?uid=098c...</td>\n",
|
||||
" <td>https://us-east1-aws.api.snapchat.com/dmd/mm?u...</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1</th>\n",
|
||||
" <td>2025-11-29 04:49:56+00:00</td>\n",
|
||||
" <td>Video</td>\n",
|
||||
" <td>Latitude, Longitude: 44.110767, -94.217705</td>\n",
|
||||
" <td>https://app.snapchat.com/dmd/memories?uid=098c...</td>\n",
|
||||
" <td>https://us-east1-aws.api.snapchat.com/dmd/mm?u...</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2</th>\n",
|
||||
" <td>2025-11-26 14:23:27+00:00</td>\n",
|
||||
" <td>Video</td>\n",
|
||||
" <td>Latitude, Longitude: 44.354908, -93.96698</td>\n",
|
||||
" <td>https://app.snapchat.com/dmd/memories?uid=098c...</td>\n",
|
||||
" <td>https://us-east1-aws.api.snapchat.com/dmd/mm?u...</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>3</th>\n",
|
||||
" <td>2025-11-26 14:18:45+00:00</td>\n",
|
||||
" <td>Video</td>\n",
|
||||
" <td>Latitude, Longitude: 44.355003, -93.96696</td>\n",
|
||||
" <td>https://app.snapchat.com/dmd/memories?uid=098c...</td>\n",
|
||||
" <td>https://us-east1-aws.api.snapchat.com/dmd/mm?u...</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>4</th>\n",
|
||||
" <td>2025-11-16 17:58:59+00:00</td>\n",
|
||||
" <td>Image</td>\n",
|
||||
" <td>Latitude, Longitude: 46.184643, -94.363495</td>\n",
|
||||
" <td>https://app.snapchat.com/dmd/memories?uid=098c...</td>\n",
|
||||
" <td>https://us-east1-aws.api.snapchat.com/dmd/mm?u...</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>5</th>\n",
|
||||
" <td>2025-11-12 02:10:34+00:00</td>\n",
|
||||
" <td>Video</td>\n",
|
||||
" <td>Latitude, Longitude: 44.354897, -93.96707</td>\n",
|
||||
" <td>https://app.snapchat.com/dmd/memories?uid=098c...</td>\n",
|
||||
" <td>https://us-east1-aws.api.snapchat.com/dmd/mm?u...</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>6</th>\n",
|
||||
" <td>2025-11-07 04:43:52+00:00</td>\n",
|
||||
" <td>Video</td>\n",
|
||||
" <td>Latitude, Longitude: 44.354797, -93.96702</td>\n",
|
||||
" <td>https://app.snapchat.com/dmd/memories?uid=098c...</td>\n",
|
||||
" <td>https://us-east1-aws.api.snapchat.com/dmd/mm?u...</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>7</th>\n",
|
||||
" <td>2025-10-27 21:50:11+00:00</td>\n",
|
||||
" <td>Video</td>\n",
|
||||
" <td>Latitude, Longitude: 44.354794, -93.967026</td>\n",
|
||||
" <td>https://app.snapchat.com/dmd/memories?uid=098c...</td>\n",
|
||||
" <td>https://us-east1-aws.api.snapchat.com/dmd/mm?u...</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>8</th>\n",
|
||||
" <td>2025-10-16 06:39:29+00:00</td>\n",
|
||||
" <td>Video</td>\n",
|
||||
" <td>Latitude, Longitude: 40.4103, -104.70806</td>\n",
|
||||
" <td>https://app.snapchat.com/dmd/memories?uid=098c...</td>\n",
|
||||
" <td>https://us-east1-aws.api.snapchat.com/dmd/mm?u...</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>9</th>\n",
|
||||
" <td>2025-10-14 20:46:28+00:00</td>\n",
|
||||
" <td>Video</td>\n",
|
||||
" <td>Latitude, Longitude: 40.432137, -104.76975</td>\n",
|
||||
" <td>https://app.snapchat.com/dmd/memories?uid=098c...</td>\n",
|
||||
" <td>https://us-east1-aws.api.snapchat.com/dmd/mm?u...</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" Date Media Type \\\n",
|
||||
"0 2025-11-29 04:50:47+00:00 Video \n",
|
||||
"1 2025-11-29 04:49:56+00:00 Video \n",
|
||||
"2 2025-11-26 14:23:27+00:00 Video \n",
|
||||
"3 2025-11-26 14:18:45+00:00 Video \n",
|
||||
"4 2025-11-16 17:58:59+00:00 Image \n",
|
||||
"5 2025-11-12 02:10:34+00:00 Video \n",
|
||||
"6 2025-11-07 04:43:52+00:00 Video \n",
|
||||
"7 2025-10-27 21:50:11+00:00 Video \n",
|
||||
"8 2025-10-16 06:39:29+00:00 Video \n",
|
||||
"9 2025-10-14 20:46:28+00:00 Video \n",
|
||||
"\n",
|
||||
" Location \\\n",
|
||||
"0 Latitude, Longitude: 44.110836, -94.217804 \n",
|
||||
"1 Latitude, Longitude: 44.110767, -94.217705 \n",
|
||||
"2 Latitude, Longitude: 44.354908, -93.96698 \n",
|
||||
"3 Latitude, Longitude: 44.355003, -93.96696 \n",
|
||||
"4 Latitude, Longitude: 46.184643, -94.363495 \n",
|
||||
"5 Latitude, Longitude: 44.354897, -93.96707 \n",
|
||||
"6 Latitude, Longitude: 44.354797, -93.96702 \n",
|
||||
"7 Latitude, Longitude: 44.354794, -93.967026 \n",
|
||||
"8 Latitude, Longitude: 40.4103, -104.70806 \n",
|
||||
"9 Latitude, Longitude: 40.432137, -104.76975 \n",
|
||||
"\n",
|
||||
" Download Link \\\n",
|
||||
"0 https://app.snapchat.com/dmd/memories?uid=098c... \n",
|
||||
"1 https://app.snapchat.com/dmd/memories?uid=098c... \n",
|
||||
"2 https://app.snapchat.com/dmd/memories?uid=098c... \n",
|
||||
"3 https://app.snapchat.com/dmd/memories?uid=098c... \n",
|
||||
"4 https://app.snapchat.com/dmd/memories?uid=098c... \n",
|
||||
"5 https://app.snapchat.com/dmd/memories?uid=098c... \n",
|
||||
"6 https://app.snapchat.com/dmd/memories?uid=098c... \n",
|
||||
"7 https://app.snapchat.com/dmd/memories?uid=098c... \n",
|
||||
"8 https://app.snapchat.com/dmd/memories?uid=098c... \n",
|
||||
"9 https://app.snapchat.com/dmd/memories?uid=098c... \n",
|
||||
"\n",
|
||||
" Media Download Url \n",
|
||||
"0 https://us-east1-aws.api.snapchat.com/dmd/mm?u... \n",
|
||||
"1 https://us-east1-aws.api.snapchat.com/dmd/mm?u... \n",
|
||||
"2 https://us-east1-aws.api.snapchat.com/dmd/mm?u... \n",
|
||||
"3 https://us-east1-aws.api.snapchat.com/dmd/mm?u... \n",
|
||||
"4 https://us-east1-aws.api.snapchat.com/dmd/mm?u... \n",
|
||||
"5 https://us-east1-aws.api.snapchat.com/dmd/mm?u... \n",
|
||||
"6 https://us-east1-aws.api.snapchat.com/dmd/mm?u... \n",
|
||||
"7 https://us-east1-aws.api.snapchat.com/dmd/mm?u... \n",
|
||||
"8 https://us-east1-aws.api.snapchat.com/dmd/mm?u... \n",
|
||||
"9 https://us-east1-aws.api.snapchat.com/dmd/mm?u... "
|
||||
]
|
||||
},
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"df.head(10)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "d8fb8fd5",
|
||||
"execution_count": null,
|
||||
"id": "71007845",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def get_join_col(row: pd.Series) -> pd.Series:\n",
|
||||
" row['file_dt'] = f\"{row['Date'].strftime('%Y-%m-%d_%H-%M-%S')}\"\n",
|
||||
" return row"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "71007845",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
" return row\n",
|
||||
"\n",
|
||||
"def extract_lat_long(row: pd.Series) -> pd.Series:\n",
|
||||
" coordinates = row[\"Location\"].split(\": \")[1]\n",
|
||||
" lat_str, long_str = coordinates.split(\", \")\n",
|
||||
@ -471,75 +292,27 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"id": "cb28dcd6",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import re\n",
|
||||
"\n",
|
||||
"def dms_to_decimal(dms_string):\n",
|
||||
" \"\"\"\n",
|
||||
" Converts a geographical coordinate string in Degrees, Minutes, Seconds (DMS)\n",
|
||||
" format (e.g., \"44 deg 6' 15.76\" N\") to a decimal degree coordinate.\n",
|
||||
"\n",
|
||||
" Args:\n",
|
||||
" dms_string (str): The DMS coordinate string.\n",
|
||||
"\n",
|
||||
" Returns:\n",
|
||||
" float: The decimal degree equivalent.\n",
|
||||
" \n",
|
||||
" Raises:\n",
|
||||
" ValueError: If the string format is not recognized.\n",
|
||||
" \"\"\"\n",
|
||||
" # Regex to capture Degrees, Minutes, Seconds, and Direction\n",
|
||||
" # Pattern: (\\d+) deg (\\d+)' ([\\d.]+)\" ([NSEW])\n",
|
||||
" # The pattern is flexible with leading zeros, extra spaces, and decimal seconds.\n",
|
||||
" match = re.search(r\"(\\d+)\\s*deg\\s*(\\d+)'\\s*([\\d.]+)\\\"\\s*([NSEW])\", dms_string, re.IGNORECASE)\n",
|
||||
"\n",
|
||||
" if not match:\n",
|
||||
" raise ValueError(f\"Coordinate format not recognized: {dms_string}\")\n",
|
||||
"\n",
|
||||
" # Extract the parts, ensuring they are converted to appropriate numerical types\n",
|
||||
" degrees = float(match.group(1))\n",
|
||||
" minutes = float(match.group(2))\n",
|
||||
" seconds = float(match.group(3))\n",
|
||||
" direction = match.group(4).upper()\n",
|
||||
"\n",
|
||||
" # Calculate the decimal degree value\n",
|
||||
" decimal_degrees = degrees + (minutes / 60) + (seconds / 3600)\n",
|
||||
"\n",
|
||||
" # Apply the sign based on the direction (S and W are negative)\n",
|
||||
" if direction in ['S', 'W']:\n",
|
||||
" decimal_degrees *= -1\n",
|
||||
" \n",
|
||||
" decimal_degrees = round(decimal_degrees, 5)\n",
|
||||
"\n",
|
||||
" return decimal_degrees"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"execution_count": null,
|
||||
"id": "97ee9100",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def convert_lat_long_to_decimal_degrees(row: pd.Series) -> pd.Series:\n",
|
||||
" # def convert(value: str) -> float:\n",
|
||||
" # if pd.isna(value):\n",
|
||||
" # return None\n",
|
||||
" # if value == '-':\n",
|
||||
" # return 0.0\n",
|
||||
" # parts = value.strip().split(' ')\n",
|
||||
" # degrees = float(parts[0])\n",
|
||||
" # minutes = float(parts[1]) / 60\n",
|
||||
" # seconds = float(parts[2]) / 3600\n",
|
||||
" # direction = parts[3]\n",
|
||||
" # decimal_degrees = degrees + minutes + seconds\n",
|
||||
" # if direction in ['S', 'W']:\n",
|
||||
" # decimal_degrees = -decimal_degrees\n",
|
||||
" # return decimal_degrees\n",
|
||||
" def dms_to_decimal(dms_string):\n",
|
||||
" match = re.search(r\"(\\d+)\\s*deg\\s*(\\d+)'\\s*([\\d.]+)\\\"\\s*([NSEW])\", dms_string, re.IGNORECASE)\n",
|
||||
" if not match:\n",
|
||||
" raise ValueError(f\"Coordinate format not recognized: {dms_string}\")\n",
|
||||
"\n",
|
||||
" degrees = float(match.group(1))\n",
|
||||
" minutes = float(match.group(2))\n",
|
||||
" seconds = float(match.group(3))\n",
|
||||
" direction = match.group(4).upper()\n",
|
||||
" decimal_degrees = degrees + (minutes / 60) + (seconds / 3600)\n",
|
||||
" if direction in ['S', 'W']:\n",
|
||||
" decimal_degrees *= -1\n",
|
||||
" decimal_degrees = round(decimal_degrees, 5)\n",
|
||||
" return decimal_degrees\n",
|
||||
" \n",
|
||||
" def convert(value: str) -> float:\n",
|
||||
" if pd.isna(value):\n",
|
||||
" return 0.0\n",
|
||||
@ -548,7 +321,7 @@
|
||||
" try:\n",
|
||||
" return dms_to_decimal(value)\n",
|
||||
" except ValueError:\n",
|
||||
" return None\n",
|
||||
" return 0.0\n",
|
||||
" \n",
|
||||
" row[\"Latitude\"] = convert(row[\"gpslatitude\"])\n",
|
||||
" row[\"Longitude\"] = convert(row[\"gpslongitude\"])\n",
|
||||
@ -557,7 +330,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"execution_count": null,
|
||||
"id": "8684b468",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
@ -566,7 +339,6 @@
|
||||
" path_parts = row['file'].split('/')\n",
|
||||
" file = path_parts[-1]\n",
|
||||
" row['filename'] = file\n",
|
||||
" \n",
|
||||
" row['extension'] = file.split('.')[-1].lower()\n",
|
||||
" row['file_dt'] = file.split('.')[0]\n",
|
||||
" return row"
|
||||
@ -711,12 +483,11 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"execution_count": null,
|
||||
"id": "543eecf8",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import pytz\n",
|
||||
"def get_datetime_for_df(row: pd.Series) -> pd.Series:\n",
|
||||
" from datetime import datetime\n",
|
||||
" try:\n",
|
||||
@ -724,32 +495,18 @@
|
||||
" if row['createdate'] != '-':\n",
|
||||
" createdate = datetime.strptime(row['createdate'], '%Y:%m:%d %H:%M:%S')\n",
|
||||
" createdate_wtz = createdate.astimezone(pytz.timezone('America/Chicago'))\n",
|
||||
" \n",
|
||||
" row['createdate_mod'] = createdate_wtz\n",
|
||||
" \n",
|
||||
" \n",
|
||||
" date = datetime.strptime(row['file_dt'], '%Y-%m-%d_%H-%M-%S')\n",
|
||||
" dt_w_tz = date.astimezone(pytz.UTC)\n",
|
||||
" # dt_w_tz = date.dtz_localize('UTC')\n",
|
||||
" # dt_w_tz = date.tz_localize('UTC')\n",
|
||||
" row['datetime'] = dt_w_tz\n",
|
||||
" row['dt_join'] = f\"{dt_w_tz}\"\n",
|
||||
" # row['datetime_str'] = dt_w_tz.strftime('%Y-%m-%d %H:%M:%S %Z')\n",
|
||||
" # row['datetime'] = datetime.strptime(row['Date'], '%Y-%m-%d %H-%M-%S')\n",
|
||||
" except ValueError:\n",
|
||||
" row['createdate_mod'] = None\n",
|
||||
" row['datetime'] = None\n",
|
||||
" row['dt_join'] = ''\n",
|
||||
" return row"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"id": "a162c993",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
" return row\n",
|
||||
"\n",
|
||||
"def get_media_type(row: pd.Series) -> pd.Series:\n",
|
||||
" row['file_type'] = 'Unknown'\n",
|
||||
" if row['extension'] == 'jpg':\n",
|
||||
@ -761,30 +518,19 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 15,
|
||||
"execution_count": null,
|
||||
"id": "b57f9726",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"data = meta.apply(get_datetime_for_df, axis=1)\n",
|
||||
"data = data.apply(get_media_type, axis=1)\n",
|
||||
"# data.head(10)"
|
||||
"joined = pd.merge(df, data, how='left', left_on='file_dt', right_on='file_dt', suffixes=('_df', '_meta'))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"id": "cea52cd0",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"joined = pd.merge(df, data, how='left', left_on='file_dt', right_on='file_dt', suffixes=('_df', '_meta'))\n",
|
||||
"# joined.head(10)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 17,
|
||||
"execution_count": null,
|
||||
"id": "3f350445",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
@ -792,7 +538,6 @@
|
||||
"col_mapper = {\n",
|
||||
" 'Date': 'correct_date_utc',\n",
|
||||
" 'Media Type': 'correct_media_type',\n",
|
||||
" # 'Location': 'correct_location',\n",
|
||||
" 'location_latitude': 'correct_latitude',\n",
|
||||
" 'location_longitude': 'correct_longitude',\n",
|
||||
" 'file': 'path',\n",
|
||||
@ -800,25 +545,14 @@
|
||||
" 'createdate': 'actual_date_cst',\n",
|
||||
" 'Latitude': 'actual_latitude',\n",
|
||||
" 'Longitude': 'actual_longitude',\n",
|
||||
" # 'extension': 'actual_extension',\n",
|
||||
" 'file_type': 'actual_media_type'\n",
|
||||
"}"
|
||||
"}\n",
|
||||
"df0 = joined[col_mapper.keys()].rename(columns=col_mapper)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 18,
|
||||
"id": "07ac5dd2",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"df0 = joined[col_mapper.keys()].rename(columns=col_mapper)\n",
|
||||
"# df0.head(10)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 19,
|
||||
"execution_count": null,
|
||||
"id": "192561b3",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
@ -829,31 +563,17 @@
|
||||
" errors += 1\n",
|
||||
" if row[\"actual_latitude\"] == 0.00 and row[\"correct_media_type\"] == \"Image\":\n",
|
||||
" if row['correct_latitude'] != 0.00:\n",
|
||||
" errors +=2\n",
|
||||
" errors +=1\n",
|
||||
" if abs(row[\"actual_longitude\"] - row[\"correct_longitude\"]) > 2.00 :\n",
|
||||
" errors +=2\n",
|
||||
" errors +=1\n",
|
||||
" if row[\"actual_date_cst\"] == '-':\n",
|
||||
" errors +=4\n",
|
||||
" errors +=1\n",
|
||||
" if pd.isna(row['path']):\n",
|
||||
" errors +=20\n",
|
||||
" errors +=1\n",
|
||||
" row[\"errors\"] = errors\n",
|
||||
" return row"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 20,
|
||||
"id": "c89101b9",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# 1 = Media Type\n",
|
||||
"# 2 = Latitude\n",
|
||||
"# 4 = Date\n",
|
||||
"# 20 = Path\n",
|
||||
" return row\n",
|
||||
"\n",
|
||||
"df1 = df0.apply(check_for_errors, axis=1)\n",
|
||||
"# df1.head(10)"
|
||||
"df1 = df0.apply(check_for_errors, axis=1)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -1032,14 +752,6 @@
|
||||
"errors.shape"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "bd5d80b3",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"679/3700 (oof)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 23,
|
||||
@ -1319,8 +1031,6 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"def change_filetype(row: pd.Series) -> pd.Series:\n",
|
||||
" try:\n",
|
||||
" if row['path'] != '-':\n",
|
||||
@ -1345,17 +1055,11 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 27,
|
||||
"execution_count": null,
|
||||
"id": "07514caa",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import pytz\n",
|
||||
"from timezonefinder import TimezoneFinder\n",
|
||||
"from datetime import datetime\n",
|
||||
"import pandas as pd # Assuming you still use pandas\n",
|
||||
"\n",
|
||||
"# Initialize the TimezoneFinder outside the loop for efficiency\n",
|
||||
"tf = TimezoneFinder()\n",
|
||||
"\n",
|
||||
"def get_localized_dt_and_offset(utc_dt, latitude, longitude):\n",
|
||||
@ -1364,32 +1068,28 @@
|
||||
" the UTC datetime object to that local time, returning the time string\n",
|
||||
" and the corresponding offset string.\n",
|
||||
" \"\"\"\n",
|
||||
" # 1. Find the IANA Timezone Name\n",
|
||||
" tz_name = tf.timezone_at(lng=longitude, lat=latitude)\n",
|
||||
" \n",
|
||||
" # Timezones\n",
|
||||
" utc_timezone = pytz.utc\n",
|
||||
" central_timezone = pytz.timezone('America/Chicago')\n",
|
||||
" mountain_timezone = pytz.timezone('America/Denver')\n",
|
||||
"\n",
|
||||
" if not tz_name or latitude == 0 or longitude == 0:\n",
|
||||
" # Fallback if no timezone is found (e.g., open ocean)\n",
|
||||
" # print(f\"⚠️ Warning: No timezone found for {latitude}, {longitude}. Defaulting to UTC.\")\n",
|
||||
" # local_tz = pytz.utc\n",
|
||||
" print(f\"⚠️ Warning: No timezone found for {latitude}, {longitude}. Defaulting to MDT/MST.\")\n",
|
||||
" local_tz = mountain_timezone\n",
|
||||
" else:\n",
|
||||
" local_tz = pytz.timezone(tz_name)\n",
|
||||
"\n",
|
||||
" # 2. Convert from UTC to the determined local timezone\n",
|
||||
" # The input 'utc_dt' must already be a timezone-aware UTC object (which it is in your current code)\n",
|
||||
" # The input 'utc_dt' must already be a timezone-aware UTC object\n",
|
||||
" dt_local = utc_dt.astimezone(local_tz)\n",
|
||||
"\n",
|
||||
" # 3. Format the local date string (for DateTimeOriginal, etc.)\n",
|
||||
" # Must use 'YYYY:MM:DD HH:MM:SS' format\n",
|
||||
" # 3. Format the local date string\n",
|
||||
" # 'YYYY:MM:DD HH:MM:SS'\n",
|
||||
" dt_str = dt_local.strftime(\"%Y:%m:%d %H:%M:%S\")\n",
|
||||
"\n",
|
||||
" # 4. Format the time zone offset string (for OffsetTimeOriginal)\n",
|
||||
" # The '+/-HHMM' string is extracted, and the colon is inserted for EXIF compatibility\n",
|
||||
" # '+/-HHMM' -> '+/-HH:MM' for EXIF compatibility\n",
|
||||
" offset_str = dt_local.strftime('%z')\n",
|
||||
" if len(offset_str) == 5:\n",
|
||||
" offset_str = offset_str[:3] + \":\" + offset_str[3:] \n",
|
||||
@ -1399,21 +1099,12 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 37,
|
||||
"execution_count": null,
|
||||
"id": "e7b8202f",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import pandas as pd\n",
|
||||
"import subprocess\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"# --- 1. Define the Update Function ---\n",
|
||||
"\n",
|
||||
"def update_exif_with_exiftool(row):\n",
|
||||
" \"\"\"\n",
|
||||
" Constructs and runs the ExifTool command for a single image.\n",
|
||||
" \"\"\"\n",
|
||||
" image_path = row['path']\n",
|
||||
" create_date = row['correct_date_utc']\n",
|
||||
" latitude = row['correct_latitude']\n",
|
||||
@ -1426,24 +1117,24 @@
|
||||
" print(f\"❌ File not found, skipping: {image_path}\")\n",
|
||||
" return\n",
|
||||
" \n",
|
||||
" utc = pytz.utc\n",
|
||||
" # utc = pytz.utc\n",
|
||||
" # Use the 'America/Chicago' zone which handles both CST (-6) and CDT (-5) automatically\n",
|
||||
" central = pytz.timezone('America/Chicago')\n",
|
||||
" # central = pytz.timezone('America/Chicago')\n",
|
||||
" # local_dt = utc_dt.astimezone(central)\n",
|
||||
" # dt_utc = utc.localize(create_date)\n",
|
||||
" dt_utc = create_date\n",
|
||||
" dt_central = dt_utc.astimezone(central)\n",
|
||||
" dt_str = dt_central.strftime(\"%Y:%m:%d %H:%M:%S\")\n",
|
||||
" # dt_central = dt_utc.astimezone(central)\n",
|
||||
" # dt_str = dt_central.strftime(\"%Y:%m:%d %H:%M:%S\")\n",
|
||||
" dt_utc_str = dt_utc.strftime(\"%Y:%m:%d %H:%M:%S\")\n",
|
||||
" \n",
|
||||
" lat_ref = 'N' if latitude >= 0 else 'S'\n",
|
||||
" lon_ref = 'E' if longitude >= 0 else 'W'\n",
|
||||
" \n",
|
||||
" # Determine the time zone offset string (e.g., \"-05:00\" or \"-06:00\")\n",
|
||||
" # This is critical for the viewing software to correctly interpret dt_str.\n",
|
||||
" offset_str = dt_central.strftime('%z')\n",
|
||||
" if len(offset_str) == 5:\n",
|
||||
" offset_str = offset_str[:3] + \":\" + offset_str[3:] # Converts -0500 to -05:00\n",
|
||||
" # # Determine the time zone offset string (e.g., \"-05:00\" or \"-06:00\")\n",
|
||||
" # # This is critical for the viewing software to correctly interpret dt_str.\n",
|
||||
" # offset_str = dt_central.strftime('%z')\n",
|
||||
" # if len(offset_str) == 5:\n",
|
||||
" # offset_str = offset_str[:3] + \":\" + offset_str[3:] # Converts -0500 to -05:00\n",
|
||||
" \n",
|
||||
" dt_str, offset_str = get_localized_dt_and_offset(\n",
|
||||
" utc_dt=dt_utc,\n",
|
||||
@ -1455,36 +1146,15 @@
|
||||
" print(f\"❌ File not found, skipping: {image_path}\")\n",
|
||||
" return\n",
|
||||
"\n",
|
||||
" # ExifTool requires the date format to be adjusted slightly, \n",
|
||||
" # and it automatically handles the degree/minute/second conversion.\n",
|
||||
" # The date MUST be in 'YYYY:MM:DD HH:MM:SS' format (same as piexif).\n",
|
||||
" \n",
|
||||
" # We use -overwrite_original to avoid creating unnecessary backup files (.jpg_original)\n",
|
||||
" # The -all= handles potential piexif conflicts by deleting all existing tags \n",
|
||||
" # before adding new ones, but is commented out for safety.\n",
|
||||
" # command = [\n",
|
||||
" # 'exiftool',\n",
|
||||
" # # '-all=', # <-- Optional: Use this to remove all existing metadata first\n",
|
||||
" # # # Set all three date tags explicitly using the Central Time value\n",
|
||||
" # f'-CreateDate={dt_str}'\n",
|
||||
" # # f'-DateTime={dt_str}'\n",
|
||||
" # # f'-DateTimeOriginal={dt_str}',\n",
|
||||
" # # f'-DateTimeDigitized={dt_str}',\n",
|
||||
" # f'-GPSLatitude={latitude}',\n",
|
||||
" # f'-GPSLongitude={longitude}',\n",
|
||||
" # f\"-GPSLatitudeRef='{lat_ref}'\",\n",
|
||||
" # f\"-GPSLongitudeRef='{lon_ref}'\",\n",
|
||||
" # '-overwrite_original',\n",
|
||||
" # image_path\n",
|
||||
" # ]\n",
|
||||
" jpg_command = [\n",
|
||||
" 'exiftool',\n",
|
||||
" # Local time (Central Time) determined by pytz\n",
|
||||
" \n",
|
||||
" # Local time\n",
|
||||
" f'-IFD0:DateTime={dt_str}',\n",
|
||||
" f'-ExifIFD:DateTimeOriginal={dt_str}',\n",
|
||||
" f'-ExifIFD:DateTimeDigitized={dt_str}',\n",
|
||||
" \n",
|
||||
" # Dynamic offset (-05:00 or -06:00) determined by pytz\n",
|
||||
" # Dynamic offset\n",
|
||||
" f'-OffsetTimeOriginal={offset_str}',\n",
|
||||
" \n",
|
||||
" # GPS tags\n",
|
||||
@ -1500,41 +1170,29 @@
|
||||
" mp4_command = [\n",
|
||||
" \"exiftool\",\n",
|
||||
" \n",
|
||||
" # --- EXIF Tags (These are often ignored by video players, but good to include) ---\n",
|
||||
" f'-IFD0:DateTime={dt_str}', # Local time\n",
|
||||
" f'-ExifIFD:DateTimeOriginal={dt_str}', # Local time\n",
|
||||
" f'-ExifIFD:DateTimeDigitized={dt_str}', # Local time\n",
|
||||
" f'-OffsetTimeOriginal={offset_str}', # Dynamic offset\n",
|
||||
" # EXIF Tags\n",
|
||||
" f'-IFD0:DateTime={dt_str}',\n",
|
||||
" f'-ExifIFD:DateTimeOriginal={dt_str}',\n",
|
||||
" f'-ExifIFD:DateTimeDigitized={dt_str}',\n",
|
||||
" f'-OffsetTimeOriginal={offset_str}',\n",
|
||||
" \n",
|
||||
" # --- QUICKTIME Tags (The critical part for MP4/MOV) ---\n",
|
||||
" # QuickTime tags should be set to UTC time string\n",
|
||||
" f\"-QuickTime:CreationDate={dt_utc_str}\", # <-- *** CHANGE 1: Use UTC Time, Different Tag Name ***\n",
|
||||
" f\"-QuickTime:TrackCreateDate={dt_utc_str}\", # <-- *** CHANGE 2: Also set Track-level tag to UTC ***\n",
|
||||
" f\"-QuickTime:MediaCreateDate={dt_utc_str}\", # <-- *** CHANGE 3: Also set Media-level tag to UTC ***\n",
|
||||
" # QuickTime tags (must be set to UTC time)\n",
|
||||
" f\"-QuickTime:CreationDate={dt_utc_str}\",\n",
|
||||
" f\"-QuickTime:TrackCreateDate={dt_utc_str}\",\n",
|
||||
" f\"-QuickTime:MediaCreateDate={dt_utc_str}\",\n",
|
||||
" \n",
|
||||
" # This is for the display TimeZone of the above tags\n",
|
||||
" f\"-QuickTime:TimeZone={offset_str}\", # Dynamic offset (e.g., -05:00)\n",
|
||||
" \n",
|
||||
" # # Generic Composite Tag (Tries to create necessary atoms)\n",
|
||||
" # f\"-GPSCoordinates={coord_str}\",\n",
|
||||
" # Not 100% sure needed, doesn't seem to hurt\n",
|
||||
" f\"-QuickTime:TimeZone={offset_str}\",\n",
|
||||
"\n",
|
||||
" # # EXIF\n",
|
||||
" # f\"-exif:gpslatitude={lat}\",\n",
|
||||
" # f\"-exif:gpslongitude={lon}\",\n",
|
||||
" # f\"-gpslatituderef={'N' if lat >= 0 else 'S'}\",\n",
|
||||
" # f\"-gpslongituderef={'E' if lon >= 0 else 'W'}\",\n",
|
||||
" \n",
|
||||
" # omposite Tag (Highly recommended for QuickTime/MP4 containers)\n",
|
||||
" # Composite Tag (Needed for location on Google Photos)\n",
|
||||
" f\"-QuickTime:GPSCoordinates={abs_latitude} {lat_ref}, {abs_longitude} {lon_ref}\",\n",
|
||||
"\n",
|
||||
" # Highly prioritized XMP Tag (Often read first by Adobe/Google)\n",
|
||||
" # XMP Tags (Adobe/Google compatibility)\n",
|
||||
" f\"-xmp:gpslatitude={latitude}\",\n",
|
||||
" f\"-xmp:gpslongitude={longitude}\",\n",
|
||||
"\n",
|
||||
" # f\"-gpslatituderef={'N' if lat >= 0 else 'S'}\",\n",
|
||||
" # f\"-gpslongituderef={'E' if lon >= 0 else 'W'}\",\n",
|
||||
"\n",
|
||||
" # QuickTime Location tags (Apple/iOS compatibility)\n",
|
||||
" # QuickTime Tags (Apple/iOS compatibility)\n",
|
||||
" f\"-QuickTime:LocationLatitude={latitude}\",\n",
|
||||
" f\"-QuickTime:LocationLongitude={longitude}\",\n",
|
||||
"\n",
|
||||
@ -1544,23 +1202,19 @@
|
||||
" f\"-GPSLatitudeRef={lat_ref}\",\n",
|
||||
" f\"-GPSLongitudeRef={lon_ref}\",\n",
|
||||
" \n",
|
||||
" # Keep overwrite and target file\n",
|
||||
" '-overwrite_original',\n",
|
||||
" image_path\n",
|
||||
" ]\n",
|
||||
"\n",
|
||||
" try:\n",
|
||||
" # Run the command\n",
|
||||
" if file_type == 'Video':\n",
|
||||
" result = subprocess.run(mp4_command, check=True, capture_output=True, text=True)\n",
|
||||
" else:\n",
|
||||
" result = subprocess.run(jpg_command, check=True, capture_output=True, text=True)\n",
|
||||
" \n",
|
||||
" # Check ExifTool output for success messages\n",
|
||||
" if \"1 image files updated\" in result.stdout:\n",
|
||||
" print(f\"✅ Updated: {image_path}\")\n",
|
||||
" else:\n",
|
||||
" # Print stderr if ExifTool reports an error (e.g., invalid data)\n",
|
||||
" print(f\"⚠️ Warning updating {image_path}: {result.stderr.strip()}\")\n",
|
||||
"\n",
|
||||
" except subprocess.CalledProcessError as e:\n",
|
||||
|
||||
Loading…
Reference in New Issue
Block a user