|
|
@@ -386,7 +386,7 @@ poetry run uvicorn app.main:app --host 0.0.0.0 --port 8000
|
|
|
|
|
|
### TODO
|
|
|
|
|
|
-- [ ] ClickHouse для хранения событий (BLE/WiFi)
|
|
|
+- [x] ClickHouse для хранения событий (BLE/WiFi)
|
|
|
- [ ] Audit logs middleware
|
|
|
- [ ] Email verification
|
|
|
- [ ] Password reset flow
|
|
|
@@ -394,6 +394,120 @@ poetry run uvicorn app.main:app --host 0.0.0.0 --port 8000
|
|
|
- [ ] WebSocket для real-time
|
|
|
- [ ] Frontend (Vue 3)
|
|
|
|
|
|
+## ClickHouse Integration
|
|
|
+
|
|
|
+### Setup
|
|
|
+
|
|
|
+**ClickHouse Version:** 25.12.1.649 (official build)
|
|
|
+
|
|
|
+**Installation:**
|
|
|
+```bash
|
|
|
+sudo systemctl start clickhouse-server
|
|
|
+clickhouse-client
|
|
|
+```
|
|
|
+
|
|
|
+**Database Creation:**
|
|
|
+```sql
|
|
|
+CREATE DATABASE mybeacon;
|
|
|
+```
|
|
|
+
|
|
|
+**Tables:**
|
|
|
+```sql
|
|
|
+-- BLE events
|
|
|
+CREATE TABLE mybeacon.ble_events (
|
|
|
+ timestamp DateTime64(3),
|
|
|
+ device_id UInt32,
|
|
|
+ device_mac String,
|
|
|
+ organization_id UInt32 DEFAULT 0,
|
|
|
+ beacon_mac String,
|
|
|
+ rssi Int8,
|
|
|
+ uuid String DEFAULT '',
|
|
|
+ major UInt16 DEFAULT 0,
|
|
|
+ minor UInt16 DEFAULT 0,
|
|
|
+ tx_power Int8 DEFAULT 0,
|
|
|
+ raw_data String DEFAULT ''
|
|
|
+) ENGINE = MergeTree()
|
|
|
+PARTITION BY toYYYYMM(timestamp)
|
|
|
+ORDER BY (organization_id, device_id, timestamp);
|
|
|
+
|
|
|
+-- WiFi events
|
|
|
+CREATE TABLE mybeacon.wifi_events (
|
|
|
+ timestamp DateTime64(3),
|
|
|
+ device_id UInt32,
|
|
|
+ device_mac String,
|
|
|
+ organization_id UInt32 DEFAULT 0,
|
|
|
+ client_mac String,
|
|
|
+ ssid String DEFAULT '',
|
|
|
+ rssi Int8,
|
|
|
+ channel UInt8 DEFAULT 0,
|
|
|
+ frame_type String DEFAULT 'probe_request',
|
|
|
+ raw_data String DEFAULT ''
|
|
|
+) ENGINE = MergeTree()
|
|
|
+PARTITION BY toYYYYMM(timestamp)
|
|
|
+ORDER BY (organization_id, device_id, timestamp);
|
|
|
+```
|
|
|
+
|
|
|
+### Implementation
|
|
|
+
|
|
|
+**Service:** `app/services/clickhouse_service.py`
|
|
|
+- `insert_ble_events()` - batch insert BLE events
|
|
|
+- `insert_wifi_events()` - batch insert WiFi events
|
|
|
+- Timestamp parsing from unix milliseconds (`ts` field)
|
|
|
+- Organization ID: 0 for unassigned devices
|
|
|
+
|
|
|
+**Integration:** `app/api/v1/events.py`
|
|
|
+- POST /ble → ClickHouse insert
|
|
|
+- POST /wifi → ClickHouse insert
|
|
|
+- Gzip compression support
|
|
|
+
|
|
|
+**Event Format (from device):**
|
|
|
+```json
|
|
|
+{
|
|
|
+ "mac": "f9:29:de:16:b4:f3",
|
|
|
+ "rssi": -64,
|
|
|
+ "ts": 1766923995934,
|
|
|
+ "type": "ibeacon",
|
|
|
+ "uuid": "b9407f30f5f8466eaff925556b57fe6d",
|
|
|
+ "major": 1,
|
|
|
+ "minor": 2
|
|
|
+}
|
|
|
+```
|
|
|
+
|
|
|
+### Performance
|
|
|
+
|
|
|
+- **Partitioning:** Monthly by timestamp (`toYYYYMM(timestamp)`)
|
|
|
+- **Ordering:** `(organization_id, device_id, timestamp)` for fast multi-tenant queries
|
|
|
+- **Batch inserts:** All events in one INSERT statement
|
|
|
+- **Data type:** DateTime64(3) for millisecond precision
|
|
|
+
|
|
|
+### Queries
|
|
|
+
|
|
|
+**Total events:**
|
|
|
+```sql
|
|
|
+SELECT count(*) FROM mybeacon.ble_events;
|
|
|
+SELECT count(*) FROM mybeacon.wifi_events;
|
|
|
+```
|
|
|
+
|
|
|
+**Unique beacons:**
|
|
|
+```sql
|
|
|
+SELECT uniq(beacon_mac) FROM mybeacon.ble_events;
|
|
|
+```
|
|
|
+
|
|
|
+**Latest events:**
|
|
|
+```sql
|
|
|
+SELECT timestamp, beacon_mac, rssi, uuid
|
|
|
+FROM mybeacon.ble_events
|
|
|
+ORDER BY timestamp DESC
|
|
|
+LIMIT 10;
|
|
|
+```
|
|
|
+
|
|
|
+**Events by organization:**
|
|
|
+```sql
|
|
|
+SELECT organization_id, count(*) as events
|
|
|
+FROM mybeacon.ble_events
|
|
|
+GROUP BY organization_id;
|
|
|
+```
|
|
|
+
|
|
|
## Performance
|
|
|
|
|
|
### Текущее состояние
|