commit f40dd26283ec774b6058070626dd101618440bbb Author: Thomas Klaehn Date: Mon Nov 2 10:17:43 2026 +0100 initial commit Signed-off-by: Thomas Klaehn diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..3b72af0 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +.env +__pycache__ diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..55e0046 --- /dev/null +++ b/.env.example @@ -0,0 +1,4 @@ +GARMIN_EMAIL=your@email.com +GARMIN_PASSWORD=yourpassword +GARMIN_IMPORT_TYPES= +TZ=Europe/Berlin diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4c49bd7 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.env diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..dda2720 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.13-slim + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY garmin_loader.py . + +ENV GARMIN_DATA_DIR=/data \ + PYTHONUNBUFFERED=1 + +VOLUME ["/data"] + +ENTRYPOINT ["python", "/app/garmin_loader.py"] +CMD ["run"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..4cf7521 --- /dev/null +++ b/README.md @@ -0,0 +1,42 @@ +# Garmin Activity Loader + +Downloads all Garmin Connect activities (original files, full history on the +first run, then new ones every 5 minutes) and hands the FIT files of selected +activity types to fit-monitor. + +Uses the unofficial Garmin Connect login via +[garminconnect](https://github.com/cyberjunky/python-garminconnect); Garmin +changes its login now and then, so a library update may be needed when it +stops working. + +## Data layout (`/srv/garmin-activity-loader/data`) + +| Path | Content | +|-----------------------|------------------------------------------------------| +| `archive//` | every activity, `__.fit` | +| `import//` | hard links to the FIT files of the selected types; mounted into fit-monitor as `/data/garmin` | +| `garmin-state.json` | downloaded activities, failed downloads, import progress | +| `.garminconnect/` | login tokens | + +## Activity type switch + +`GARMIN_IMPORT_TYPES` in `.env` is a comma-separated list of Garmin type keys. +A parent key includes its sub types (`cycling` covers `road_biking`, +`virtual_ride`, …). Empty means archive only. List the keys, with `*` marking +the currently selected ones: + +```sh +docker compose run --rm garmin-activity-loader types +``` + +After changing the switch, restart the container; `import/` is updated on the +next cycle. Removing a type only removes its links, activities fit-monitor has +already imported stay in the database. + +## Setup + +```sh +cp .env.example .env # fill in credentials and GARMIN_IMPORT_TYPES +docker compose run --rm garmin-activity-loader login # asks for the MFA code if needed +docker compose up -d +``` diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..9f5a0b7 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,30 @@ +services: + garmin-activity-loader: + build: . + container_name: garmin-activity-loader + restart: unless-stopped + # Same owner as the data folder on the host. + user: "1000:1000" + environment: + # Required: Garmin Connect credentials (use .env file) + - GARMIN_EMAIL=${GARMIN_EMAIL} + - GARMIN_PASSWORD=${GARMIN_PASSWORD} + + # Activity types handed to fit-monitor (comma-separated Garmin type + # keys; a parent key like "cycling" includes its sub types). Everything + # else is only archived. List the keys with: + # docker compose run --rm garmin-activity-loader types + - GARMIN_IMPORT_TYPES=${GARMIN_IMPORT_TYPES:-} + + - GARMIN_POLL_INTERVAL=${GARMIN_POLL_INTERVAL:-5m} + - GARMIN_LOG_LEVEL=${GARMIN_LOG_LEVEL:-info} + - TZ=${TZ:-Europe/Berlin} + volumes: + # archive/ holds every activity, import/ the selected types (mounted + # into fit-monitor), plus state file and login tokens. + - /srv/garmin-activity-loader/data:/data + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" diff --git a/garmin_loader.py b/garmin_loader.py new file mode 100644 index 0000000..50d29d1 --- /dev/null +++ b/garmin_loader.py @@ -0,0 +1,317 @@ +"""Garmin Connect activity loader. + +Downloads the original files of all Garmin Connect activities into +data/archive// and mirrors the FIT files of the activity types +selected in GARMIN_IMPORT_TYPES into data/import// as hard links. +fit-monitor only watches data/import, so the switch decides what reaches the +database while the archive always holds everything. + +Commands: + run (default) sync now, then every GARMIN_POLL_INTERVAL + login interactive login, asks for the MFA code if the account needs one + types list Garmin activity type keys usable in GARMIN_IMPORT_TYPES +""" + +import io +import json +import logging +import os +import re +import signal +import sys +import threading +import time +import zipfile +from datetime import datetime, timezone +from pathlib import Path + +from garminconnect import ( + Garmin, + GarminConnectAuthenticationError, + GarminConnectConnectionError, + GarminConnectNotFoundError, + GarminConnectTooManyRequestsError, +) + +DATA_DIR = Path(os.environ.get("GARMIN_DATA_DIR", "/data")) +ARCHIVE_DIR = DATA_DIR / "archive" +IMPORT_DIR = DATA_DIR / "import" +STATE_FILE = DATA_DIR / "garmin-state.json" +TOKEN_DIR = DATA_DIR / ".garminconnect" + +PAGE_SIZE = 100 +# Pause between file downloads so the initial full-history import doesn't +# hammer Garmin Connect. +DOWNLOAD_DELAY_SECONDS = float(os.environ.get("GARMIN_DOWNLOAD_DELAY", "1")) + +log = logging.getLogger("garmin-loader") +stop = threading.Event() + + +def parse_interval(value: str) -> int: + m = re.fullmatch(r"\s*(\d+)\s*([smh]?)\s*", value) + if not m: + raise ValueError(f"invalid interval {value!r}, expected e.g. 30s, 5m, 1h") + return int(m.group(1)) * {"": 1, "s": 1, "m": 60, "h": 3600}[m.group(2)] + + +def import_types() -> set[str]: + raw = os.environ.get("GARMIN_IMPORT_TYPES", "") + return {t.strip() for t in raw.split(",") if t.strip()} + + +def sanitize(name: str) -> str: + name = re.sub(r"[^a-zA-Z0-9_-]", "", name.replace(" ", "-")) + return re.sub(r"-+", "-", name).strip("-")[:50] or "activity" + + +class State: + """Downloaded activities and history-import progress, kept as JSON.""" + + def __init__(self, path: Path): + self.path = path + self.activities: dict[str, dict] = {} + # Activities whose download failed, retried on every sync. + self.failed: dict[str, dict] = {} + self.backfill_complete = False + # typeKey -> parent typeKey, so "cycling" also selects road_biking etc. + self.type_parents: dict[str, str] = {} + if path.exists(): + data = json.loads(path.read_text()) + self.activities = data.get("activities", {}) + self.failed = data.get("failed", {}) + self.backfill_complete = data.get("backfill_complete", False) + self.type_parents = data.get("type_parents", {}) + + def save(self): + tmp = self.path.with_suffix(".tmp") + tmp.write_text(json.dumps({ + "activities": self.activities, + "failed": self.failed, + "backfill_complete": self.backfill_complete, + "type_parents": self.type_parents, + "last_check": datetime.now(timezone.utc).isoformat(), + }, indent=1)) + tmp.replace(self.path) + + +def connect(prompt_mfa=None) -> Garmin: + api = Garmin( + os.environ.get("GARMIN_EMAIL") or None, + os.environ.get("GARMIN_PASSWORD") or None, + prompt_mfa=prompt_mfa, + ) + # Reuses the stored tokens (refreshed and re-saved by the library) and only + # falls back to a credential login when they're missing or rejected. + api.login(str(TOKEN_DIR)) + return api + + +def refresh_activity_types(api: Garmin, state: State): + types = api.get_activity_types() + by_id = {t["typeId"]: t["typeKey"] for t in types} + state.type_parents = { + t["typeKey"]: by_id.get(t.get("parentTypeId"), "") for t in types + } + + +def is_imported(type_key: str, state: State, selected: set[str]) -> bool: + return type_key in selected or state.type_parents.get(type_key) in selected + + +def download(api: Garmin, state: State, activity: dict) -> bool: + """Store one activity's original file(s) in the archive. Returns False + when the download failed and should be retried later.""" + activity_id = str(activity["activityId"]) + type_key = activity.get("activityType", {}).get("typeKey") or "unknown" + start = activity.get("startTimeLocal", "") + name = activity.get("activityName") or "" + base = f"{activity_id}_{start[:10]}_{sanitize(name)}" + + try: + data = api.download_activity(activity_id, dl_fmt=Garmin.ActivityDownloadFormat.ORIGINAL) + except GarminConnectNotFoundError: + # Manually entered activities have no file to download. + data = b"" + except (GarminConnectConnectionError, OSError) as e: + log.warning("download failed for %s (%s): %s", activity_id, type_key, e) + state.failed[activity_id] = activity + return False + + # "Original" is a zip holding the uploaded file(s), normally one .fit. + members: list[tuple[str, bytes]] = [] + if data[:2] == b"PK": + with zipfile.ZipFile(io.BytesIO(data)) as z: + for info in z.infolist(): + if not info.is_dir(): + members.append((Path(info.filename).suffix.lower(), z.read(info))) + elif data: + members.append((".fit", data)) + + target_dir = ARCHIVE_DIR / type_key + target_dir.mkdir(parents=True, exist_ok=True) + files = [] + for i, (suffix, content) in enumerate(members): + target = target_dir / f"{base}{'' if i == 0 else f'_{i}'}{suffix}" + tmp = target.with_name(target.name + ".tmp") + tmp.write_bytes(content) + tmp.replace(target) + files.append(str(target.relative_to(DATA_DIR))) + + state.activities[activity_id] = { + "type": type_key, + "name": name, + "start": start, + "files": files, + } + state.failed.pop(activity_id, None) + log.info("downloaded %s %s (%s)%s", activity_id, start, type_key, + "" if files else " - no file") + return True + + +def sync(api: Garmin, state: State): + refresh_activity_types(api, state) + + # Newest first. The first run walks the whole history; later runs stop at + # the first page that reaches activities we already have. + offset = 0 + downloaded = 0 + while not stop.is_set(): + page = api.get_activities(offset, PAGE_SIZE) + if not page: + if not state.backfill_complete: + state.backfill_complete = True + log.info("full history import complete: %d activities", len(state.activities)) + break + new = [a for a in page if str(a["activityId"]) not in state.activities] + for activity in new: + if stop.is_set(): + break + if download(api, state, activity): + downloaded += 1 + state.save() + time.sleep(DOWNLOAD_DELAY_SECONDS) + if state.backfill_complete and len(new) < len(page): + break + offset += len(page) + + for activity in list(state.failed.values()): + if stop.is_set(): + break + if download(api, state, activity): + downloaded += 1 + time.sleep(DOWNLOAD_DELAY_SECONDS) + + state.save() + log.info("sync done: %d new, %d failed, %d total", downloaded, len(state.failed), + len(state.activities)) + + +def reconcile_import(state: State): + """Make data/import mirror the FIT files of the selected types. Removing a + type from the switch only removes its links; activities fit-monitor has + already imported stay in the database.""" + selected = import_types() + IMPORT_DIR.mkdir(parents=True, exist_ok=True) + linked = removed = 0 + + for type_dir in sorted(ARCHIVE_DIR.iterdir()) if ARCHIVE_DIR.exists() else []: + if not type_dir.is_dir() or not is_imported(type_dir.name, state, selected): + continue + out_dir = IMPORT_DIR / type_dir.name + for src in type_dir.glob("*.fit"): + dst = out_dir / src.name + if dst.exists(): + continue + out_dir.mkdir(exist_ok=True) + # Link under a non-.fit name first: fit-monitor reacts to the + # create event, so the .fit name must only appear complete. + tmp = out_dir / (src.name + ".tmp") + tmp.unlink(missing_ok=True) + os.link(src, tmp) + tmp.replace(dst) + linked += 1 + + for out_dir in IMPORT_DIR.iterdir(): + if out_dir.is_dir() and not is_imported(out_dir.name, state, selected): + for f in out_dir.iterdir(): + f.unlink() + removed += 1 + out_dir.rmdir() + + if linked or removed: + log.info("import folder updated: %d linked, %d removed", linked, removed) + + +def run(): + interval = parse_interval(os.environ.get("GARMIN_POLL_INTERVAL", "5m")) + state = State(STATE_FILE) + + selected = import_types() + log.info("importing types: %s", ", ".join(sorted(selected)) or "(none, archive only)") + unknown = selected - set(state.type_parents) if state.type_parents else set() + if unknown: + log.warning("unknown activity types in GARMIN_IMPORT_TYPES: %s", ", ".join(sorted(unknown))) + + api = None + while not stop.is_set(): + wait = interval + try: + if api is None: + api = connect() + sync(api, state) + except GarminConnectAuthenticationError as e: + # No retry storm: repeated failed logins can get the account locked. + log.error("login failed (%s) - if the account uses MFA, run the login command", e) + api = None + wait = max(interval, 3600) + except GarminConnectTooManyRequestsError: + log.warning("rate limited by Garmin Connect, pausing for an hour") + wait = max(interval, 3600) + except Exception: + log.exception("sync failed") + finally: + state.save() + reconcile_import(state) + stop.wait(wait) + + +def login(): + connect(prompt_mfa=lambda: input("MFA code: ").strip()) + log.info("login ok, tokens saved in %s", TOKEN_DIR) + + +def types(): + api = connect() + state = State(STATE_FILE) + refresh_activity_types(api, state) + state.save() + selected = import_types() + for key in sorted(state.type_parents, key=lambda k: (state.type_parents[k], k)): + parent = state.type_parents[key] + mark = "*" if is_imported(key, state, selected) else " " + print(f"{mark} {key}" + (f" (in {parent})" if parent else "")) + + +def main(): + logging.basicConfig( + level=os.environ.get("GARMIN_LOG_LEVEL", "info").upper(), + format="%(asctime)s %(levelname)s %(message)s", + ) + # fit-monitor runs as a different user and must be able to read the files. + os.umask(0o022) + DATA_DIR.mkdir(parents=True, exist_ok=True) + + signal.signal(signal.SIGTERM, lambda *_: stop.set()) + signal.signal(signal.SIGINT, lambda *_: stop.set()) + + command = sys.argv[1] if len(sys.argv) > 1 else "run" + commands = {"run": run, "login": login, "types": types} + if command not in commands: + sys.exit(f"unknown command {command!r}, expected one of: {', '.join(commands)}") + commands[command]() + + +if __name__ == "__main__": + main() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..dfb9750 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +garminconnect==0.3.18