v0.1.3 — DeFlock via Overpass + per-source health UI

The cdn.deflock.me CDN is gated behind Cloudflare bot mitigation that mobile HTTP clients can't pass. The live deflock-app Flutter client abandoned that path; it POSTs Overpass-QL queries directly to overpass.deflock.org (with overpass-api.de as a fallback). Verified by hitting the same endpoint from curl — 22 ALPRs returned for the Springfield VA bbox, matching the user's screenshot of the working app. DeflockClient rewrite: - POST [out:json][timeout:25];(node[surveillance][type=ALPR](bbox););out body; - 5 km half-width bbox around the user - 24h on-disk cache keyed by 0.05° grid cell (revisits don't refetch) - Returns sealed FetchResult: Success(points) | Failed(reason) DeflockScanner update: - Replaces 20° tile concept with distance-based refetch (1.5 km threshold) - Records SourceHealth on each fetch outcome Waze: reCAPTCHA gating confirmed. WazeClient.fetchPoliceNear now returns sealed FetchResult; WazeScanner records SourceHealth.FAILED with "Upstream blocked (HTTP 403)" so the user sees why no Waze data is flowing instead of silent zeros. New fusion/SourceHealth.kt — per-source MutableStateFlow registry, record(source, ok, message) + reset() called on service start/stop. UI: SourceRow in the bottom-sheet drill-down now shows the health message in orange when status = FAILED instead of "no detections". versionCode 3 → 4, versionName 0.1.2 → 0.1.3.
2026-04-28 21:36:47 -04:00
parent 6c57297f58
commit 00584f58c9
9 changed files with 254 additions and 77 deletions
@@ -12,8 +12,8 @@ android {
        applicationId = "org.soulstone.overwatch"
        minSdk = 26
        targetSdk = 35
-        versionCode = 3
-        versionName = "0.1.2"
+        versionCode = 4
+        versionName = "0.1.3"
    }

    buildTypes {
@@ -0,0 +1,63 @@
+package org.soulstone.overwatch.fusion
+
+import kotlinx.coroutines.flow.MutableStateFlow
+import kotlinx.coroutines.flow.StateFlow
+import kotlinx.coroutines.flow.asStateFlow
+
+/**
+ * Per-source upstream-health registry.
+ *
+ * Network sources (DEFLOCK, WAZE) record OK/FAILED so the UI can distinguish
+ * "scanned, found nothing" from "couldn't reach the data source." BLE/WIFI
+ * are radio-only and don't currently report; they default to UNKNOWN, which
+ * the UI treats the same as OK.
+ */
+object SourceHealth {
+
+    enum class Status { UNKNOWN, OK, FAILED }
+
+    data class Health(
+        val status: Status = Status.UNKNOWN,
+        val lastFetchMs: Long = 0L,
+        /** Short reason shown in the UI when status = FAILED. */
+        val message: String? = null
+    )
+
+    private val _ble = MutableStateFlow(Health())
+    private val _wifi = MutableStateFlow(Health())
+    private val _deflock = MutableStateFlow(Health())
+    private val _waze = MutableStateFlow(Health())
+
+    val ble: StateFlow<Health> = _ble.asStateFlow()
+    val wifi: StateFlow<Health> = _wifi.asStateFlow()
+    val deflock: StateFlow<Health> = _deflock.asStateFlow()
+    val waze: StateFlow<Health> = _waze.asStateFlow()
+
+    fun flowFor(source: DetectionSource): StateFlow<Health> = when (source) {
+        DetectionSource.BLE -> ble
+        DetectionSource.WIFI -> wifi
+        DetectionSource.DEFLOCK -> deflock
+        DetectionSource.WAZE -> waze
+    }
+
+    fun record(source: DetectionSource, ok: Boolean, message: String? = null) {
+        val target = when (source) {
+            DetectionSource.BLE -> _ble
+            DetectionSource.WIFI -> _wifi
+            DetectionSource.DEFLOCK -> _deflock
+            DetectionSource.WAZE -> _waze
+        }
+        target.value = Health(
+            status = if (ok) Status.OK else Status.FAILED,
+            lastFetchMs = System.currentTimeMillis(),
+            message = message
+        )
+    }
+
+    fun reset() {
+        _ble.value = Health()
+        _wifi.value = Health()
+        _deflock.value = Health()
+        _waze.value = Health()
+    }
+}
@@ -5,31 +5,39 @@ import android.util.Log
 import java.io.File
 import java.net.HttpURLConnection
 import java.net.URL
+import java.net.URLEncoder
+import kotlin.math.floor
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.withContext
-import kotlin.math.floor
-import org.json.JSONArray
+import org.json.JSONObject

 /**
- * Fetches DeFlock ALPR tile data from the public CDN, with a 24h on-disk cache.
+ * Fetches DeFlock ALPR data from the Overpass API (matching the live deflock-app
+ * Flutter client). The earlier `cdn.deflock.me/regions/...json` path is now
+ * gated behind Cloudflare bot mitigation that we cannot pass from a mobile HTTP
+ * client.
 *
- * Tile scheme (from REFERENCES/deflock/serverless/alpr_cache):
- *   tile_lat = floor(lat / 20) * 20
- *   tile_lon = floor(lon / 20) * 20
- *   url      = https://cdn.deflock.me/regions/{tile_lat}/{tile_lon}.json
- *   body     = JSON array of { id: number, lat: number, lon: number, tags: {…} }
- *
- * 20° tiles → ≤16 tiles cover the entire globe; one user typically only ever touches one.
+ * Strategy:
+ *  - POST an Overpass-QL query for `man_made=surveillance + surveillance:type=ALPR`
+ *    inside a small bbox around the user.
+ *  - Try `overpass.deflock.org` first (less rate-limited for this use case),
+ *    fall back to public `overpass-api.de`.
+ *  - Cache the JSON response on disk by 0.05° grid cell (24h TTL). Revisits to
+ *    the same cell don't re-hit the API.
 */
 class DeflockClient(context: Context) {

    companion object {
        private const val TAG = "DeflockClient"
-        private const val TILE_SIZE_DEG = 20
+        private const val FETCH_RADIUS_DEG = 0.05  // ~5.5 km half-width bbox
        private const val CACHE_TTL_MS = 24L * 60L * 60L * 1000L
-        private const val CDN_BASE = "https://cdn.deflock.me/regions"
        private const val USER_AGENT = "OVERWATCH/0.1 (+github.com/KaraZajac/OVERWATCH)"
-        private const val TIMEOUT_MS = 15_000
+        private const val TIMEOUT_MS = 30_000
+        private const val OVERPASS_QUERY_TIMEOUT_S = 25
+        private val ENDPOINTS = listOf(
+            "https://overpass.deflock.org/api/interpreter",
+            "https://overpass-api.de/api/interpreter"
+        )
    }

    data class AlprPoint(
@@ -40,63 +48,94 @@ class DeflockClient(context: Context) {
        val manufacturer: String? = null
    )

-    data class TileKey(val tileLat: Int, val tileLon: Int) {
-        fun fileName() = "deflock_${tileLat}_${tileLon}.json"
+    /** Outcome of a fetch — distinguishes "no ALPRs in area" from "couldn't reach the API." */
+    sealed class FetchResult {
+        data class Success(val points: List<AlprPoint>) : FetchResult()
+        data class Failed(val reason: String) : FetchResult()
    }

    private val cacheDir: File = File(context.cacheDir, "deflock").apply { mkdirs() }

-    fun tileFor(lat: Double, lon: Double): TileKey = TileKey(
-        tileLat = floor(lat / TILE_SIZE_DEG).toInt() * TILE_SIZE_DEG,
-        tileLon = floor(lon / TILE_SIZE_DEG).toInt() * TILE_SIZE_DEG
-    )
-
-    /** Returns parsed ALPR points for the tile; empty list on any failure (logged). */
-    suspend fun fetchTile(tile: TileKey): List<AlprPoint> = withContext(Dispatchers.IO) {
-        val cached = cachedJson(tile)
+    suspend fun fetchAround(lat: Double, lon: Double): FetchResult = withContext(Dispatchers.IO) {
+        val key = cacheKeyFor(lat, lon)
+        val cached = cachedJson(key)
        if (cached != null) {
-            return@withContext parseSafely(cached)
+            Log.d(TAG, "Cache hit for $key")
+            return@withContext FetchResult.Success(parseSafely(cached))
+        }
+        val south = lat - FETCH_RADIUS_DEG
+        val north = lat + FETCH_RADIUS_DEG
+        val west = lon - FETCH_RADIUS_DEG
+        val east = lon + FETCH_RADIUS_DEG
+        val query = buildQuery(south, west, north, east)
+        val (body, lastError) = downloadFromAny(query)
+        if (body == null) {
+            return@withContext FetchResult.Failed(lastError ?: "Network error")
        }
-        val downloaded = downloadTile(tile) ?: return@withContext emptyList()
        try {
-            File(cacheDir, tile.fileName()).writeText(downloaded)
+            File(cacheDir, "$key.json").writeText(body)
        } catch (e: Exception) {
-            Log.w(TAG, "Failed to write tile cache for $tile: ${e.message}")
+            Log.w(TAG, "Failed to write cache for $key: ${e.message}")
        }
-        parseSafely(downloaded)
+        FetchResult.Success(parseSafely(body))
    }

-    private fun cachedJson(tile: TileKey): String? {
-        val f = File(cacheDir, tile.fileName())
+    private fun cacheKeyFor(lat: Double, lon: Double): String {
+        // 0.05° grid cell. Two consecutive points within the same cell get the
+        // same cache key, so micro-movements don't refetch.
+        val latStep = floor(lat / FETCH_RADIUS_DEG).toInt()
+        val lonStep = floor(lon / FETCH_RADIUS_DEG).toInt()
+        return "deflock_${latStep}_${lonStep}"
+    }
+
+    private fun cachedJson(key: String): String? {
+        val f = File(cacheDir, "$key.json")
        if (!f.exists()) return null
-        val age = System.currentTimeMillis() - f.lastModified()
-        if (age > CACHE_TTL_MS) return null
+        if (System.currentTimeMillis() - f.lastModified() > CACHE_TTL_MS) return null
        return try { f.readText() } catch (e: Exception) { null }
    }

-    private fun downloadTile(tile: TileKey): String? {
-        val url = URL("$CDN_BASE/${tile.tileLat}/${tile.tileLon}.json")
+    private fun buildQuery(south: Double, west: Double, north: Double, east: Double): String =
+        "[out:json][timeout:$OVERPASS_QUERY_TIMEOUT_S];" +
+            "(node[\"man_made\"=\"surveillance\"][\"surveillance:type\"=\"ALPR\"]" +
+            "($south,$west,$north,$east););out body;"
+
+    /** Try each endpoint in order until one returns 2xx. Returns body + last error message. */
+    private fun downloadFromAny(query: String): Pair<String?, String?> {
+        var lastError: String? = null
+        for (endpoint in ENDPOINTS) {
+            val (body, err) = postQuery(endpoint, query)
+            if (body != null) return body to null
+            lastError = err
+        }
+        return null to lastError
+    }
+
+    private fun postQuery(endpoint: String, query: String): Pair<String?, String?> {
+        val url = URL(endpoint)
        val conn = (url.openConnection() as HttpURLConnection).apply {
            connectTimeout = TIMEOUT_MS
            readTimeout = TIMEOUT_MS
-            requestMethod = "GET"
+            requestMethod = "POST"
+            doOutput = true
            setRequestProperty("User-Agent", USER_AGENT)
+            setRequestProperty("Content-Type", "application/x-www-form-urlencoded")
            setRequestProperty("Accept", "application/json")
        }
        return try {
+            val payload = "data=" + URLEncoder.encode(query, "UTF-8")
+            conn.outputStream.use { it.write(payload.toByteArray()) }
            val code = conn.responseCode
-            if (code == 404) {
-                Log.i(TAG, "Tile $tile not present on CDN (no ALPRs in this region)")
-                ""  // cache the empty result by writing an empty string
-            } else if (code in 200..299) {
-                conn.inputStream.bufferedReader().use { it.readText() }
+            if (code in 200..299) {
+                val body = conn.inputStream.bufferedReader().use { it.readText() }
+                body to null
            } else {
-                Log.w(TAG, "CDN returned $code for $tile")
-                null
+                Log.w(TAG, "$endpoint returned $code")
+                null to "HTTP $code"
            }
        } catch (e: Exception) {
-            Log.w(TAG, "Download failed for $tile: ${e.message}")
-            null
+            Log.w(TAG, "$endpoint failed: ${e.message}")
+            null to (e.message ?: e.javaClass.simpleName)
        } finally {
            conn.disconnect()
        }
@@ -105,17 +144,19 @@ class DeflockClient(context: Context) {
    private fun parseSafely(json: String): List<AlprPoint> {
        if (json.isBlank()) return emptyList()
        return try {
-            val arr = JSONArray(json)
-            val out = ArrayList<AlprPoint>(arr.length())
-            for (i in 0 until arr.length()) {
-                val o = arr.getJSONObject(i)
-                val lat = o.optDouble("lat")
-                val lon = o.optDouble("lon")
+            val root = JSONObject(json)
+            val elements = root.optJSONArray("elements") ?: return emptyList()
+            val out = ArrayList<AlprPoint>(elements.length())
+            for (i in 0 until elements.length()) {
+                val el = elements.optJSONObject(i) ?: continue
+                if (el.optString("type") != "node") continue
+                val lat = el.optDouble("lat")
+                val lon = el.optDouble("lon")
                if (lat.isNaN() || lon.isNaN()) continue
-                val tags = o.optJSONObject("tags")
+                val tags = el.optJSONObject("tags")
                out.add(
                    AlprPoint(
-                        id = o.optLong("id", 0L),
+                        id = el.optLong("id", 0L),
                        lat = lat,
                        lon = lon,
                        operator = tags?.optString("operator")?.ifBlank { null }
@@ -129,7 +170,7 @@ class DeflockClient(context: Context) {
            }
            out
        } catch (e: Exception) {
-            Log.w(TAG, "Failed to parse tile JSON: ${e.message}")
+            Log.w(TAG, "Failed to parse Overpass response: ${e.message}")
            emptyList()
        }
    }
@@ -11,17 +11,16 @@ import org.soulstone.overwatch.fusion.ConfidenceEngine
 import org.soulstone.overwatch.fusion.DetectionEvent
 import org.soulstone.overwatch.fusion.DetectionSource
 import org.soulstone.overwatch.fusion.DetectionStore
+import org.soulstone.overwatch.fusion.SourceHealth

 /**
 * DeFlock orchestrator.
 *
- * Subscribes to [LocationProvider]; for each new fix, looks up the matching 20° tile
- * (loaded from [DeflockClient] cache or downloaded once / 24h) and submits a
- * detection event for every ALPR within [PROXIMITY_M].
- *
- * Tile-boundary edge case: at lat ≈ tile_lat or lon ≈ tile_lon ±0.002°, ALPRs across
- * the boundary won't be visible until the user crosses it. Acceptable for v0.1 — a
- * 5-tile fetch (current + 4 neighbours) is a polish item.
+ * Subscribes to [LocationProvider]; when the user has moved more than
+ * [REFETCH_THRESHOLD_M] from the last fetch center (or there is no last
+ * center), runs an Overpass query via [DeflockClient] for the surrounding
+ * 5-km bbox. For each cached ALPR within [proximityMeters], submits a
+ * detection event.
 */
 class DeflockScanner(
    private val store: DetectionStore,
@@ -32,10 +31,12 @@ class DeflockScanner(

    companion object {
        private const val TAG = "DeflockScanner"
+        private const val REFETCH_THRESHOLD_M = 1500f
    }

    private var job: Job? = null
-    private var lastTile: DeflockClient.TileKey? = null
+    private var lastFetchLat: Double? = null
+    private var lastFetchLon: Double? = null
    private var cachedPoints: List<DeflockClient.AlprPoint> = emptyList()

    fun start(scope: CoroutineScope): Boolean {
@@ -52,17 +53,36 @@ class DeflockScanner(
    fun stop() {
        job?.cancel()
        job = null
-        lastTile = null
+        lastFetchLat = null
+        lastFetchLon = null
        cachedPoints = emptyList()
        Log.i(TAG, "DeflockScanner stopped")
    }

    private suspend fun handleFix(fix: Location) {
-        val tile = client.tileFor(fix.latitude, fix.longitude)
-        if (tile != lastTile) {
-            cachedPoints = client.fetchTile(tile)
-            lastTile = tile
-            Log.i(TAG, "Loaded tile $tile with ${cachedPoints.size} ALPRs")
+        if (shouldRefetch(fix)) {
+            when (val result = client.fetchAround(fix.latitude, fix.longitude)) {
+                is DeflockClient.FetchResult.Success -> {
+                    cachedPoints = result.points
+                    lastFetchLat = fix.latitude
+                    lastFetchLon = fix.longitude
+                    SourceHealth.record(DetectionSource.DEFLOCK, ok = true)
+                    Log.i(
+                        TAG,
+                        "Loaded ${cachedPoints.size} ALPRs around " +
+                            "(${fix.latitude}, ${fix.longitude})"
+                    )
+                }
+                is DeflockClient.FetchResult.Failed -> {
+                    SourceHealth.record(
+                        DetectionSource.DEFLOCK,
+                        ok = false,
+                        message = "Overpass unreachable: ${result.reason}"
+                    )
+                    Log.w(TAG, "Overpass fetch failed: ${result.reason}")
+                    // Keep using cachedPoints (may be empty on first failure).
+                }
+            }
        }
        if (cachedPoints.isEmpty()) return

@@ -91,4 +111,12 @@ class DeflockScanner(
            )
        }
    }
+
+    private fun shouldRefetch(fix: Location): Boolean {
+        val lat = lastFetchLat ?: return true
+        val lon = lastFetchLon ?: return true
+        val out = FloatArray(1)
+        Location.distanceBetween(lat, lon, fix.latitude, fix.longitude, out)
+        return out[0] > REFETCH_THRESHOLD_M
+    }
 }
@@ -48,7 +48,13 @@ class WazeClient {
        val reportedBy: String?
    )

-    suspend fun fetchPoliceNear(lat: Double, lon: Double): List<Alert> = withContext(Dispatchers.IO) {
+    /** Outcome — distinguishes "no police alerts in area" from "couldn't reach Waze." */
+    sealed class FetchResult {
+        data class Success(val alerts: List<Alert>) : FetchResult()
+        data class Failed(val reason: String) : FetchResult()
+    }
+
+    suspend fun fetchPoliceNear(lat: Double, lon: Double): FetchResult = withContext(Dispatchers.IO) {
        val top = lat + BBOX_HALF_DEG
        val bottom = lat - BBOX_HALF_DEG
        val left = lon - BBOX_HALF_DEG
@@ -67,15 +73,22 @@ class WazeClient {
        }
        try {
            val code = conn.responseCode
+            if (code == 403) {
+                // Waze added reCAPTCHA gating to live-map in 2025/2026; mobile
+                // clients can no longer hit this endpoint without browser-level
+                // automation. Surface this distinctly so the UI can say so.
+                Log.w(TAG, "Waze returned 403 (upstream reCAPTCHA gating)")
+                return@withContext FetchResult.Failed("Upstream blocked (HTTP 403)")
+            }
            if (code !in 200..299) {
                Log.w(TAG, "Waze returned $code")
-                return@withContext emptyList()
+                return@withContext FetchResult.Failed("HTTP $code")
            }
            val body = conn.inputStream.bufferedReader().use { it.readText() }
-            parsePolice(body)
+            FetchResult.Success(parsePolice(body))
        } catch (e: Exception) {
            Log.w(TAG, "Waze fetch failed: ${e.message}")
-            emptyList()
+            FetchResult.Failed(e.message ?: e.javaClass.simpleName)
        } finally {
            conn.disconnect()
        }
@@ -12,6 +12,7 @@ import org.soulstone.overwatch.fusion.ConfidenceEngine
 import org.soulstone.overwatch.fusion.DetectionEvent
 import org.soulstone.overwatch.fusion.DetectionSource
 import org.soulstone.overwatch.fusion.DetectionStore
+import org.soulstone.overwatch.fusion.SourceHealth

 /**
 * Polls Waze every 60s for live POLICE alerts in a small bounding box around the
@@ -59,7 +60,21 @@ class WazeScanner(
    }

    private suspend fun pollOnce(fix: Location) {
-        val alerts = client.fetchPoliceNear(fix.latitude, fix.longitude)
+        val result = client.fetchPoliceNear(fix.latitude, fix.longitude)
+        val alerts = when (result) {
+            is WazeClient.FetchResult.Success -> {
+                SourceHealth.record(DetectionSource.WAZE, ok = true)
+                result.alerts
+            }
+            is WazeClient.FetchResult.Failed -> {
+                SourceHealth.record(
+                    DetectionSource.WAZE,
+                    ok = false,
+                    message = "Waze unreachable: ${result.reason}"
+                )
+                return
+            }
+        }
        if (alerts.isEmpty()) return
        val now = System.currentTimeMillis()
        val limit = proximityMeters()
@@ -24,6 +24,7 @@ import org.soulstone.overwatch.R
 import org.soulstone.overwatch.data.location.LocationProvider
 import org.soulstone.overwatch.data.settings.Settings
 import org.soulstone.overwatch.fusion.DetectionStore
+import org.soulstone.overwatch.fusion.SourceHealth
 import org.soulstone.overwatch.scan.BleScanner
 import org.soulstone.overwatch.scan.DeflockClient
 import org.soulstone.overwatch.scan.DeflockScanner
@@ -116,6 +117,7 @@ class DetectionService : LifecycleService() {

    private fun beginScanning() {
        if (_running.value) return
+        SourceHealth.reset()
        startInForeground()
        if (settings.bleEnabled.value) {
            bleStarted = bleScanner.start()
@@ -157,6 +159,7 @@ class DetectionService : LifecycleService() {
        if (wazeStarted) { wazeScanner.stop(); wazeStarted = false }
        locationProvider.stop()
        store.clear()
+        SourceHealth.reset()
        pruneJob?.cancel()
        pruneJob = null
        _running.value = false
@@ -33,6 +33,7 @@ import androidx.compose.material3.ModalBottomSheet
 import androidx.compose.material3.Text
 import androidx.compose.material3.rememberModalBottomSheetState
 import androidx.compose.runtime.Composable
+import androidx.compose.runtime.collectAsState
 import androidx.compose.runtime.getValue
 import androidx.compose.runtime.mutableStateOf
 import androidx.compose.runtime.remember
@@ -50,6 +51,7 @@ import androidx.compose.ui.unit.sp
 import kotlinx.coroutines.launch
 import org.soulstone.overwatch.fusion.DetectionEvent
 import org.soulstone.overwatch.fusion.DetectionSource
+import org.soulstone.overwatch.fusion.SourceHealth
 import org.soulstone.overwatch.fusion.ThreatLevel
 import org.soulstone.overwatch.ui.theme.ThreatColors

@@ -272,6 +274,9 @@ private fun SourcesPanel(events: List<DetectionEvent>) {

@Composable
 private fun SourceRow(source: DetectionSource, events: List<DetectionEvent>) {
+    val health by SourceHealth.flowFor(source).collectAsState()
+    val unreachable = health.status == SourceHealth.Status.FAILED
+
    Card(
        modifier = Modifier.fillMaxWidth(),
        colors = CardDefaults.cardColors(
@@ -294,6 +299,7 @@ private fun SourceRow(source: DetectionSource, events: List<DetectionEvent>) {
                )
                val maxScore = events.maxOfOrNull { it.score } ?: 0
                val statusColor = when {
+                    unreachable -> MaterialTheme.colorScheme.onSurfaceVariant
                    maxScore >= ThreatLevel.RED.minScore -> ThreatColors.Red
                    maxScore >= ThreatLevel.ORANGE.minScore -> ThreatColors.Orange
                    maxScore >= ThreatLevel.YELLOW.minScore -> ThreatColors.Yellow
@@ -306,7 +312,15 @@ private fun SourceRow(source: DetectionSource, events: List<DetectionEvent>) {
                        .background(statusColor)
                )
            }
-            if (events.isEmpty()) {
+            if (unreachable) {
+                Spacer(Modifier.height(4.dp))
+                Text(
+                    text = health.message ?: "Source unavailable",
+                    color = ThreatColors.Orange,
+                    fontSize = 11.sp,
+                    fontFamily = FontFamily.Monospace
+                )
+            } else if (events.isEmpty()) {
                Spacer(Modifier.height(4.dp))
                Text(
                    text = "no detections",