initial commit

Signed-off-by: Thomas Klaehn <tkl@blackfinn.de>

.dockerignore

@@ -0,0 +1,2 @@
+.env
+__pycache__

.env.example

@@ -0,0 +1,4 @@
+GARMIN_EMAIL=your@email.com
+GARMIN_PASSWORD=yourpassword
+GARMIN_IMPORT_TYPES=
+TZ=Europe/Berlin

.gitignore

@@ -0,0 +1 @@
+.env

Dockerfile

@@ -0,0 +1,16 @@
+FROM python:3.13-slim
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY garmin_loader.py .
+
+ENV GARMIN_DATA_DIR=/data \
+    PYTHONUNBUFFERED=1
+
+VOLUME ["/data"]
+
+ENTRYPOINT ["python", "/app/garmin_loader.py"]
+CMD ["run"]

README.md

@@ -0,0 +1,42 @@
+# Garmin Activity Loader
+
+Downloads all Garmin Connect activities (original files, full history on the
+first run, then new ones every 5 minutes) and hands the FIT files of selected
+activity types to fit-monitor.
+
+Uses the unofficial Garmin Connect login via
+[garminconnect](https://github.com/cyberjunky/python-garminconnect); Garmin
+changes its login now and then, so a library update may be needed when it
+stops working.
+
+## Data layout (`/srv/garmin-activity-loader/data`)
+
+| Path                  | Content                                              |
+|-----------------------|------------------------------------------------------|
+| `archive/<typeKey>/`  | every activity, `<id>_<date>_<name>.fit`             |
+| `import/<typeKey>/`   | hard links to the FIT files of the selected types; mounted into fit-monitor as `/data/garmin` |
+| `garmin-state.json`   | downloaded activities, failed downloads, import progress |
+| `.garminconnect/`     | login tokens                                         |
+
+## Activity type switch
+
+`GARMIN_IMPORT_TYPES` in `.env` is a comma-separated list of Garmin type keys.
+A parent key includes its sub types (`cycling` covers `road_biking`,
+`virtual_ride`, …). Empty means archive only. List the keys, with `*` marking
+the currently selected ones:
+
+```sh
+docker compose run --rm garmin-activity-loader types
+```
+
+After changing the switch, restart the container; `import/` is updated on the
+next cycle. Removing a type only removes its links, activities fit-monitor has
+already imported stay in the database.
+
+## Setup
+
+```sh
+cp .env.example .env    # fill in credentials and GARMIN_IMPORT_TYPES
+docker compose run --rm garmin-activity-loader login   # asks for the MFA code if needed
+docker compose up -d
+```

docker-compose.yml

@@ -0,0 +1,30 @@
+services:
+  garmin-activity-loader:
+    build: .
+    container_name: garmin-activity-loader
+    restart: unless-stopped
+    # Same owner as the data folder on the host.
+    user: "1000:1000"
+    environment:
+      # Required: Garmin Connect credentials (use .env file)
+      - GARMIN_EMAIL=${GARMIN_EMAIL}
+      - GARMIN_PASSWORD=${GARMIN_PASSWORD}
+
+      # Activity types handed to fit-monitor (comma-separated Garmin type
+      # keys; a parent key like "cycling" includes its sub types). Everything
+      # else is only archived. List the keys with:
+      #   docker compose run --rm garmin-activity-loader types
+      - GARMIN_IMPORT_TYPES=${GARMIN_IMPORT_TYPES:-}
+
+      - GARMIN_POLL_INTERVAL=${GARMIN_POLL_INTERVAL:-5m}
+      - GARMIN_LOG_LEVEL=${GARMIN_LOG_LEVEL:-info}
+      - TZ=${TZ:-Europe/Berlin}
+    volumes:
+      # archive/ holds every activity, import/ the selected types (mounted
+      # into fit-monitor), plus state file and login tokens.
+      - /srv/garmin-activity-loader/data:/data
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: "3"

garmin_loader.py

@@ -0,0 +1,317 @@
+"""Garmin Connect activity loader.
+
+Downloads the original files of all Garmin Connect activities into
+data/archive/<typeKey>/ and mirrors the FIT files of the activity types
+selected in GARMIN_IMPORT_TYPES into data/import/<typeKey>/ as hard links.
+fit-monitor only watches data/import, so the switch decides what reaches the
+database while the archive always holds everything.
+
+Commands:
+  run    (default) sync now, then every GARMIN_POLL_INTERVAL
+  login  interactive login, asks for the MFA code if the account needs one
+  types  list Garmin activity type keys usable in GARMIN_IMPORT_TYPES
+"""
+
+import io
+import json
+import logging
+import os
+import re
+import signal
+import sys
+import threading
+import time
+import zipfile
+from datetime import datetime, timezone
+from pathlib import Path
+
+from garminconnect import (
+    Garmin,
+    GarminConnectAuthenticationError,
+    GarminConnectConnectionError,
+    GarminConnectNotFoundError,
+    GarminConnectTooManyRequestsError,
+)
+
+DATA_DIR = Path(os.environ.get("GARMIN_DATA_DIR", "/data"))
+ARCHIVE_DIR = DATA_DIR / "archive"
+IMPORT_DIR = DATA_DIR / "import"
+STATE_FILE = DATA_DIR / "garmin-state.json"
+TOKEN_DIR = DATA_DIR / ".garminconnect"
+
+PAGE_SIZE = 100
+# Pause between file downloads so the initial full-history import doesn't
+# hammer Garmin Connect.
+DOWNLOAD_DELAY_SECONDS = float(os.environ.get("GARMIN_DOWNLOAD_DELAY", "1"))
+
+log = logging.getLogger("garmin-loader")
+stop = threading.Event()
+
+
+def parse_interval(value: str) -> int:
+    m = re.fullmatch(r"\s*(\d+)\s*([smh]?)\s*", value)
+    if not m:
+        raise ValueError(f"invalid interval {value!r}, expected e.g. 30s, 5m, 1h")
+    return int(m.group(1)) * {"": 1, "s": 1, "m": 60, "h": 3600}[m.group(2)]
+
+
+def import_types() -> set[str]:
+    raw = os.environ.get("GARMIN_IMPORT_TYPES", "")
+    return {t.strip() for t in raw.split(",") if t.strip()}
+
+
+def sanitize(name: str) -> str:
+    name = re.sub(r"[^a-zA-Z0-9_-]", "", name.replace(" ", "-"))
+    return re.sub(r"-+", "-", name).strip("-")[:50] or "activity"
+
+
+class State:
+    """Downloaded activities and history-import progress, kept as JSON."""
+
+    def __init__(self, path: Path):
+        self.path = path
+        self.activities: dict[str, dict] = {}
+        # Activities whose download failed, retried on every sync.
+        self.failed: dict[str, dict] = {}
+        self.backfill_complete = False
+        # typeKey -> parent typeKey, so "cycling" also selects road_biking etc.
+        self.type_parents: dict[str, str] = {}
+        if path.exists():
+            data = json.loads(path.read_text())
+            self.activities = data.get("activities", {})
+            self.failed = data.get("failed", {})
+            self.backfill_complete = data.get("backfill_complete", False)
+            self.type_parents = data.get("type_parents", {})
+
+    def save(self):
+        tmp = self.path.with_suffix(".tmp")
+        tmp.write_text(json.dumps({
+            "activities": self.activities,
+            "failed": self.failed,
+            "backfill_complete": self.backfill_complete,
+            "type_parents": self.type_parents,
+            "last_check": datetime.now(timezone.utc).isoformat(),
+        }, indent=1))
+        tmp.replace(self.path)
+
+
+def connect(prompt_mfa=None) -> Garmin:
+    api = Garmin(
+        os.environ.get("GARMIN_EMAIL") or None,
+        os.environ.get("GARMIN_PASSWORD") or None,
+        prompt_mfa=prompt_mfa,
+    )
+    # Reuses the stored tokens (refreshed and re-saved by the library) and only
+    # falls back to a credential login when they're missing or rejected.
+    api.login(str(TOKEN_DIR))
+    return api
+
+
+def refresh_activity_types(api: Garmin, state: State):
+    types = api.get_activity_types()
+    by_id = {t["typeId"]: t["typeKey"] for t in types}
+    state.type_parents = {
+        t["typeKey"]: by_id.get(t.get("parentTypeId"), "") for t in types
+    }
+
+
+def is_imported(type_key: str, state: State, selected: set[str]) -> bool:
+    return type_key in selected or state.type_parents.get(type_key) in selected
+
+
+def download(api: Garmin, state: State, activity: dict) -> bool:
+    """Store one activity's original file(s) in the archive. Returns False
+    when the download failed and should be retried later."""
+    activity_id = str(activity["activityId"])
+    type_key = activity.get("activityType", {}).get("typeKey") or "unknown"
+    start = activity.get("startTimeLocal", "")
+    name = activity.get("activityName") or ""
+    base = f"{activity_id}_{start[:10]}_{sanitize(name)}"
+
+    try:
+        data = api.download_activity(activity_id, dl_fmt=Garmin.ActivityDownloadFormat.ORIGINAL)
+    except GarminConnectNotFoundError:
+        # Manually entered activities have no file to download.
+        data = b""
+    except (GarminConnectConnectionError, OSError) as e:
+        log.warning("download failed for %s (%s): %s", activity_id, type_key, e)
+        state.failed[activity_id] = activity
+        return False
+
+    # "Original" is a zip holding the uploaded file(s), normally one .fit.
+    members: list[tuple[str, bytes]] = []
+    if data[:2] == b"PK":
+        with zipfile.ZipFile(io.BytesIO(data)) as z:
+            for info in z.infolist():
+                if not info.is_dir():
+                    members.append((Path(info.filename).suffix.lower(), z.read(info)))
+    elif data:
+        members.append((".fit", data))
+
+    target_dir = ARCHIVE_DIR / type_key
+    target_dir.mkdir(parents=True, exist_ok=True)
+    files = []
+    for i, (suffix, content) in enumerate(members):
+        target = target_dir / f"{base}{'' if i == 0 else f'_{i}'}{suffix}"
+        tmp = target.with_name(target.name + ".tmp")
+        tmp.write_bytes(content)
+        tmp.replace(target)
+        files.append(str(target.relative_to(DATA_DIR)))
+
+    state.activities[activity_id] = {
+        "type": type_key,
+        "name": name,
+        "start": start,
+        "files": files,
+    }
+    state.failed.pop(activity_id, None)
+    log.info("downloaded %s %s (%s)%s", activity_id, start, type_key,
+             "" if files else " - no file")
+    return True
+
+
+def sync(api: Garmin, state: State):
+    refresh_activity_types(api, state)
+
+    # Newest first. The first run walks the whole history; later runs stop at
+    # the first page that reaches activities we already have.
+    offset = 0
+    downloaded = 0
+    while not stop.is_set():
+        page = api.get_activities(offset, PAGE_SIZE)
+        if not page:
+            if not state.backfill_complete:
+                state.backfill_complete = True
+                log.info("full history import complete: %d activities", len(state.activities))
+            break
+        new = [a for a in page if str(a["activityId"]) not in state.activities]
+        for activity in new:
+            if stop.is_set():
+                break
+            if download(api, state, activity):
+                downloaded += 1
+            state.save()
+            time.sleep(DOWNLOAD_DELAY_SECONDS)
+        if state.backfill_complete and len(new) < len(page):
+            break
+        offset += len(page)
+
+    for activity in list(state.failed.values()):
+        if stop.is_set():
+            break
+        if download(api, state, activity):
+            downloaded += 1
+        time.sleep(DOWNLOAD_DELAY_SECONDS)
+
+    state.save()
+    log.info("sync done: %d new, %d failed, %d total", downloaded, len(state.failed),
+             len(state.activities))
+
+
+def reconcile_import(state: State):
+    """Make data/import mirror the FIT files of the selected types. Removing a
+    type from the switch only removes its links; activities fit-monitor has
+    already imported stay in the database."""
+    selected = import_types()
+    IMPORT_DIR.mkdir(parents=True, exist_ok=True)
+    linked = removed = 0
+
+    for type_dir in sorted(ARCHIVE_DIR.iterdir()) if ARCHIVE_DIR.exists() else []:
+        if not type_dir.is_dir() or not is_imported(type_dir.name, state, selected):
+            continue
+        out_dir = IMPORT_DIR / type_dir.name
+        for src in type_dir.glob("*.fit"):
+            dst = out_dir / src.name
+            if dst.exists():
+                continue
+            out_dir.mkdir(exist_ok=True)
+            # Link under a non-.fit name first: fit-monitor reacts to the
+            # create event, so the .fit name must only appear complete.
+            tmp = out_dir / (src.name + ".tmp")
+            tmp.unlink(missing_ok=True)
+            os.link(src, tmp)
+            tmp.replace(dst)
+            linked += 1
+
+    for out_dir in IMPORT_DIR.iterdir():
+        if out_dir.is_dir() and not is_imported(out_dir.name, state, selected):
+            for f in out_dir.iterdir():
+                f.unlink()
+                removed += 1
+            out_dir.rmdir()
+
+    if linked or removed:
+        log.info("import folder updated: %d linked, %d removed", linked, removed)
+
+
+def run():
+    interval = parse_interval(os.environ.get("GARMIN_POLL_INTERVAL", "5m"))
+    state = State(STATE_FILE)
+
+    selected = import_types()
+    log.info("importing types: %s", ", ".join(sorted(selected)) or "(none, archive only)")
+    unknown = selected - set(state.type_parents) if state.type_parents else set()
+    if unknown:
+        log.warning("unknown activity types in GARMIN_IMPORT_TYPES: %s", ", ".join(sorted(unknown)))
+
+    api = None
+    while not stop.is_set():
+        wait = interval
+        try:
+            if api is None:
+                api = connect()
+            sync(api, state)
+        except GarminConnectAuthenticationError as e:
+            # No retry storm: repeated failed logins can get the account locked.
+            log.error("login failed (%s) - if the account uses MFA, run the login command", e)
+            api = None
+            wait = max(interval, 3600)
+        except GarminConnectTooManyRequestsError:
+            log.warning("rate limited by Garmin Connect, pausing for an hour")
+            wait = max(interval, 3600)
+        except Exception:
+            log.exception("sync failed")
+        finally:
+            state.save()
+        reconcile_import(state)
+        stop.wait(wait)
+
+
+def login():
+    connect(prompt_mfa=lambda: input("MFA code: ").strip())
+    log.info("login ok, tokens saved in %s", TOKEN_DIR)
+
+
+def types():
+    api = connect()
+    state = State(STATE_FILE)
+    refresh_activity_types(api, state)
+    state.save()
+    selected = import_types()
+    for key in sorted(state.type_parents, key=lambda k: (state.type_parents[k], k)):
+        parent = state.type_parents[key]
+        mark = "*" if is_imported(key, state, selected) else " "
+        print(f"{mark} {key}" + (f"  (in {parent})" if parent else ""))
+
+
+def main():
+    logging.basicConfig(
+        level=os.environ.get("GARMIN_LOG_LEVEL", "info").upper(),
+        format="%(asctime)s %(levelname)s %(message)s",
+    )
+    # fit-monitor runs as a different user and must be able to read the files.
+    os.umask(0o022)
+    DATA_DIR.mkdir(parents=True, exist_ok=True)
+
+    signal.signal(signal.SIGTERM, lambda *_: stop.set())
+    signal.signal(signal.SIGINT, lambda *_: stop.set())
+
+    command = sys.argv[1] if len(sys.argv) > 1 else "run"
+    commands = {"run": run, "login": login, "types": types}
+    if command not in commands:
+        sys.exit(f"unknown command {command!r}, expected one of: {', '.join(commands)}")
+    commands[command]()
+
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1 @@
+garminconnect==0.3.18