From 567d81127699c05b6bc7bc7982986a60025cb8d3 Mon Sep 17 00:00:00 2001 From: Josh Flatt Date: Wed, 17 Dec 2025 23:48:13 -0600 Subject: [PATCH] discovery --- notebooks/check_stats.ipynb | 565 ++++++++++++++++++++++++++++++++++++ notebooks/make_df.ipynb | 2 +- 2 files changed, 566 insertions(+), 1 deletion(-) create mode 100644 notebooks/check_stats.ipynb diff --git a/notebooks/check_stats.ipynb b/notebooks/check_stats.ipynb new file mode 100644 index 0000000..ec7ddec --- /dev/null +++ b/notebooks/check_stats.ipynb @@ -0,0 +1,565 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "id": "ca3c6247", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import re\n", + "import pytz\n", + "import os\n", + "from timezonefinder import TimezoneFinder\n", + "from datetime import datetime\n", + "import subprocess" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "a5c9c72a", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "file", + "rawType": "object", + "type": "string" + }, + { + "name": "createdate", + "rawType": "object", + "type": "string" + }, + { + "name": "gpslatitude", + "rawType": "object", + "type": "string" + }, + { + "name": "gpslongitude", + "rawType": "object", + "type": "string" + } + ], + "ref": "61c03abd-c7f2-49b6-86f7-198f85172a02", + "rows": [ + [ + "0", + "../downloads/2019-04-05_23-27-17.jpg", + "-", + "40 deg 24' 14.76\" N", + "104 deg 44' 21.52\" W" + ], + [ + "1", + "../downloads/2024-10-23_00-29-09.jpg", + "-", + "44 deg 6' 15.77\" N", + "94 deg 13' 10.52\" W" + ], + [ + "2", + "../downloads/2025-02-16_00-09-41.mp4", + "2025:02:16 00:10:01", + "44 deg 21' 17.71\" N", + "93 deg 58' 1.20\" W" + ], + [ + "3", + "../downloads/2023-03-10_22-11-18.jpg", + "-", + "40 deg 24' 39.13\" N", + "104 deg 41' 33.90\" W" + ], + [ + "4", + "../downloads/2024-02-24_22-18-07.jpg", + "-", + "40 deg 25' 17.08\" N", + "104 deg 45' 22.75\" W" + ] + ], + "shape": { + "columns": 4, + "rows": 5 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
filecreatedategpslatitudegpslongitude
0../downloads/2019-04-05_23-27-17.jpg-40 deg 24' 14.76\" N104 deg 44' 21.52\" W
1../downloads/2024-10-23_00-29-09.jpg-44 deg 6' 15.77\" N94 deg 13' 10.52\" W
2../downloads/2025-02-16_00-09-41.mp42025:02:16 00:10:0144 deg 21' 17.71\" N93 deg 58' 1.20\" W
3../downloads/2023-03-10_22-11-18.jpg-40 deg 24' 39.13\" N104 deg 41' 33.90\" W
4../downloads/2024-02-24_22-18-07.jpg-40 deg 25' 17.08\" N104 deg 45' 22.75\" W
\n", + "
" + ], + "text/plain": [ + " file createdate \\\n", + "0 ../downloads/2019-04-05_23-27-17.jpg - \n", + "1 ../downloads/2024-10-23_00-29-09.jpg - \n", + "2 ../downloads/2025-02-16_00-09-41.mp4 2025:02:16 00:10:01 \n", + "3 ../downloads/2023-03-10_22-11-18.jpg - \n", + "4 ../downloads/2024-02-24_22-18-07.jpg - \n", + "\n", + " gpslatitude gpslongitude \n", + "0 40 deg 24' 14.76\" N 104 deg 44' 21.52\" W \n", + "1 44 deg 6' 15.77\" N 94 deg 13' 10.52\" W \n", + "2 44 deg 21' 17.71\" N 93 deg 58' 1.20\" W \n", + "3 40 deg 24' 39.13\" N 104 deg 41' 33.90\" W \n", + "4 40 deg 25' 17.08\" N 104 deg 45' 22.75\" W " + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "metadata = pd.read_csv(\"../temp/filemetadata.csv\", sep=\"#\")\n", + "metadata[\"file\"] = f\"../\" + metadata[\"file\"]\n", + "metadata.head(5)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "5bdf3ec9", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "file", + "rawType": "object", + "type": "string" + }, + { + "name": "createdate", + "rawType": "object", + "type": "unknown" + }, + { + "name": "gpslatitude", + "rawType": "object", + "type": "string" + }, + { + "name": "gpslongitude", + "rawType": "object", + "type": "string" + } + ], + "ref": "a616d395-4ab3-42a6-bae2-9bc6e3869132", + "rows": [ + [ + "0", + "../downloads/2019-04-05_23-27-17.jpg", + null, + "40 deg 24' 14.76\" N", + "104 deg 44' 21.52\" W" + ], + [ + "1", + "../downloads/2024-10-23_00-29-09.jpg", + null, + "44 deg 6' 15.77\" N", + "94 deg 13' 10.52\" W" + ], + [ + "2", + "../downloads/2025-02-16_00-09-41.mp4", + "2025:02:16 00:10:01", + "44 deg 21' 17.71\" N", + "93 deg 58' 1.20\" W" + ], + [ + "3", + "../downloads/2023-03-10_22-11-18.jpg", + null, + "40 deg 24' 39.13\" N", + "104 deg 41' 33.90\" W" + ], + [ + "4", + "../downloads/2024-02-24_22-18-07.jpg", + null, + "40 deg 25' 17.08\" N", + "104 deg 45' 22.75\" W" + ], + [ + "5", + "../downloads/2020-10-24_03-27-27.mp4", + "2020:10:24 03:27:17", + "40 deg 24' 37.62\" N", + "104 deg 42' 28.73\" W" + ], + [ + "6", + "../downloads/2020-11-04_04-40-23.mp4", + "2020:11:04 04:40:30", + "40 deg 25' 55.63\" N", + "104 deg 46' 10.16\" W" + ], + [ + "7", + "../downloads/2017-06-09_22-44-18.jpg", + null, + "39 deg 50' 49.74\" N", + "104 deg 40' 29.57\" W" + ], + [ + "8", + "../downloads/2018-12-26_06-49-23.mp4", + "2018:12:26 06:49:31", + "40 deg 25' 56.21\" N", + "104 deg 46' 11.24\" W" + ], + [ + "9", + "../downloads/2024-07-25_04-57-56.mp4", + "2024:07:25 04:58:13", + "44 deg 21' 17.17\" N", + "93 deg 57' 57.82\" W" + ] + ], + "shape": { + "columns": 4, + "rows": 10 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
filecreatedategpslatitudegpslongitude
0../downloads/2019-04-05_23-27-17.jpg<NA>40 deg 24' 14.76\" N104 deg 44' 21.52\" W
1../downloads/2024-10-23_00-29-09.jpg<NA>44 deg 6' 15.77\" N94 deg 13' 10.52\" W
2../downloads/2025-02-16_00-09-41.mp42025:02:16 00:10:0144 deg 21' 17.71\" N93 deg 58' 1.20\" W
3../downloads/2023-03-10_22-11-18.jpg<NA>40 deg 24' 39.13\" N104 deg 41' 33.90\" W
4../downloads/2024-02-24_22-18-07.jpg<NA>40 deg 25' 17.08\" N104 deg 45' 22.75\" W
5../downloads/2020-10-24_03-27-27.mp42020:10:24 03:27:1740 deg 24' 37.62\" N104 deg 42' 28.73\" W
6../downloads/2020-11-04_04-40-23.mp42020:11:04 04:40:3040 deg 25' 55.63\" N104 deg 46' 10.16\" W
7../downloads/2017-06-09_22-44-18.jpg<NA>39 deg 50' 49.74\" N104 deg 40' 29.57\" W
8../downloads/2018-12-26_06-49-23.mp42018:12:26 06:49:3140 deg 25' 56.21\" N104 deg 46' 11.24\" W
9../downloads/2024-07-25_04-57-56.mp42024:07:25 04:58:1344 deg 21' 17.17\" N93 deg 57' 57.82\" W
\n", + "
" + ], + "text/plain": [ + " file createdate \\\n", + "0 ../downloads/2019-04-05_23-27-17.jpg \n", + "1 ../downloads/2024-10-23_00-29-09.jpg \n", + "2 ../downloads/2025-02-16_00-09-41.mp4 2025:02:16 00:10:01 \n", + "3 ../downloads/2023-03-10_22-11-18.jpg \n", + "4 ../downloads/2024-02-24_22-18-07.jpg \n", + "5 ../downloads/2020-10-24_03-27-27.mp4 2020:10:24 03:27:17 \n", + "6 ../downloads/2020-11-04_04-40-23.mp4 2020:11:04 04:40:30 \n", + "7 ../downloads/2017-06-09_22-44-18.jpg \n", + "8 ../downloads/2018-12-26_06-49-23.mp4 2018:12:26 06:49:31 \n", + "9 ../downloads/2024-07-25_04-57-56.mp4 2024:07:25 04:58:13 \n", + "\n", + " gpslatitude gpslongitude \n", + "0 40 deg 24' 14.76\" N 104 deg 44' 21.52\" W \n", + "1 44 deg 6' 15.77\" N 94 deg 13' 10.52\" W \n", + "2 44 deg 21' 17.71\" N 93 deg 58' 1.20\" W \n", + "3 40 deg 24' 39.13\" N 104 deg 41' 33.90\" W \n", + "4 40 deg 25' 17.08\" N 104 deg 45' 22.75\" W \n", + "5 40 deg 24' 37.62\" N 104 deg 42' 28.73\" W \n", + "6 40 deg 25' 55.63\" N 104 deg 46' 10.16\" W \n", + "7 39 deg 50' 49.74\" N 104 deg 40' 29.57\" W \n", + "8 40 deg 25' 56.21\" N 104 deg 46' 11.24\" W \n", + "9 44 deg 21' 17.17\" N 93 deg 57' 57.82\" W " + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "with_nas = metadata.replace(\"-\", pd.NA)\n", + "with_nas.head(10)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "56ded30f", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "file", + "rawType": "object", + "type": "string" + }, + { + "name": "createdate", + "rawType": "object", + "type": "unknown" + }, + { + "name": "gpslatitude", + "rawType": "object", + "type": "unknown" + }, + { + "name": "gpslongitude", + "rawType": "object", + "type": "unknown" + } + ], + "ref": "1dc672e3-1cac-4c30-b026-e92e8de1cde0", + "rows": [ + [ + "2326", + "../downloads/2023-11-29_17-14-20.jpg", + null, + null, + null + ] + ], + "shape": { + "columns": 4, + "rows": 1 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
filecreatedategpslatitudegpslongitude
2326../downloads/2023-11-29_17-14-20.jpg<NA><NA><NA>
\n", + "
" + ], + "text/plain": [ + " file createdate gpslatitude gpslongitude\n", + "2326 ../downloads/2023-11-29_17-14-20.jpg " + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "missing_coords = with_nas[with_nas[\"gpslatitude\"].isna()]\n", + "missing_coords.head(10)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/make_df.ipynb b/notebooks/make_df.ipynb index fe47bb7..a172c1d 100644 --- a/notebooks/make_df.ipynb +++ b/notebooks/make_df.ipynb @@ -1595,7 +1595,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.6" + "version": "3.13.11" } }, "nbformat": 4,