Add initial MQTT scrubber service scaffold
This commit is contained in:
@@ -0,0 +1,158 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"mqqt-scrubber/internal/model"
|
||||
)
|
||||
|
||||
var invalidNameCharacters = regexp.MustCompile(`[^a-z0-9_]+`)
|
||||
var acronymBoundary = regexp.MustCompile(`([A-Z]+)([A-Z][a-z])`)
|
||||
var camelBoundary = regexp.MustCompile(`([a-z0-9])([A-Z])`)
|
||||
|
||||
var tasmotaTimeLayouts = []string{
|
||||
time.RFC3339Nano,
|
||||
time.RFC3339,
|
||||
"2006-01-02T15:04:05",
|
||||
}
|
||||
|
||||
func ParseTasmota(message model.RawMessage) ([]model.Record, error) {
|
||||
parts := strings.Split(message.Topic, "/")
|
||||
if len(parts) != 3 {
|
||||
return nil, fmt.Errorf("unsupported topic shape: %s", message.Topic)
|
||||
}
|
||||
if parts[0] != "tele" {
|
||||
return nil, fmt.Errorf("unsupported topic root: %s", message.Topic)
|
||||
}
|
||||
|
||||
measurement := "tasmota_" + sanitizeName(parts[2])
|
||||
tags := map[string]string{
|
||||
"device": sanitizeDeviceName(parts[1]),
|
||||
"message_type": sanitizeName(parts[2]),
|
||||
"source": "tasmota",
|
||||
}
|
||||
|
||||
if strings.EqualFold(parts[2], "LWT") {
|
||||
return []model.Record{parseLWT(message, measurement, tags)}, nil
|
||||
}
|
||||
|
||||
var payload map[string]any
|
||||
if err := json.Unmarshal(message.Payload, &payload); err != nil {
|
||||
return nil, fmt.Errorf("invalid json payload: %w", err)
|
||||
}
|
||||
|
||||
fields := flattenPayload(payload, nil)
|
||||
delete(fields, "time")
|
||||
if len(fields) == 0 {
|
||||
return nil, fmt.Errorf("no usable fields in payload for topic %s", message.Topic)
|
||||
}
|
||||
|
||||
timestamp := parsePayloadTimestamp(payload, message.ReceivedAt)
|
||||
|
||||
record := model.Record{
|
||||
Measurement: measurement,
|
||||
Tags: tags,
|
||||
Fields: fields,
|
||||
Timestamp: timestamp,
|
||||
}
|
||||
|
||||
return []model.Record{record}, nil
|
||||
}
|
||||
|
||||
func parseLWT(message model.RawMessage, measurement string, tags map[string]string) model.Record {
|
||||
state := strings.TrimSpace(string(message.Payload))
|
||||
return model.Record{
|
||||
Measurement: measurement,
|
||||
Tags: tags,
|
||||
Fields: map[string]any{
|
||||
"state": state,
|
||||
"online": strings.EqualFold(state, "Online"),
|
||||
},
|
||||
Timestamp: message.ReceivedAt,
|
||||
}
|
||||
}
|
||||
|
||||
func parsePayloadTimestamp(payload map[string]any, fallback time.Time) time.Time {
|
||||
rawTime, ok := payload["Time"].(string)
|
||||
if !ok || strings.TrimSpace(rawTime) == "" {
|
||||
return fallback
|
||||
}
|
||||
|
||||
for _, layout := range tasmotaTimeLayouts {
|
||||
var (
|
||||
parsed time.Time
|
||||
err error
|
||||
)
|
||||
|
||||
if layout == "2006-01-02T15:04:05" {
|
||||
parsed, err = time.ParseInLocation(layout, rawTime, time.UTC)
|
||||
} else {
|
||||
parsed, err = time.Parse(layout, rawTime)
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
return parsed
|
||||
}
|
||||
}
|
||||
|
||||
return fallback
|
||||
}
|
||||
|
||||
func flattenPayload(payload map[string]any, prefix []string) map[string]any {
|
||||
result := make(map[string]any)
|
||||
|
||||
keys := make([]string, 0, len(payload))
|
||||
for key := range payload {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
for _, key := range keys {
|
||||
value := payload[key]
|
||||
nameParts := append(prefix, sanitizeName(key))
|
||||
|
||||
switch typed := value.(type) {
|
||||
case map[string]any:
|
||||
nested := flattenPayload(typed, nameParts)
|
||||
for nestedKey, nestedValue := range nested {
|
||||
result[nestedKey] = nestedValue
|
||||
}
|
||||
case float64, bool, string:
|
||||
result[strings.Join(nameParts, "_")] = typed
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func sanitizeName(value string) string {
|
||||
normalized := strings.TrimSpace(value)
|
||||
normalized = strings.ReplaceAll(normalized, "-", "_")
|
||||
normalized = strings.ReplaceAll(normalized, " ", "_")
|
||||
normalized = acronymBoundary.ReplaceAllString(normalized, `${1}_${2}`)
|
||||
normalized = camelBoundary.ReplaceAllString(normalized, `${1}_${2}`)
|
||||
normalized = strings.ToLower(normalized)
|
||||
normalized = invalidNameCharacters.ReplaceAllString(normalized, "_")
|
||||
normalized = strings.Trim(normalized, "_")
|
||||
if normalized == "" {
|
||||
return "unknown"
|
||||
}
|
||||
return normalized
|
||||
}
|
||||
|
||||
func sanitizeDeviceName(value string) string {
|
||||
normalized := strings.ToLower(strings.TrimSpace(value))
|
||||
normalized = strings.ReplaceAll(normalized, "-", "_")
|
||||
normalized = strings.ReplaceAll(normalized, " ", "_")
|
||||
normalized = invalidNameCharacters.ReplaceAllString(normalized, "_")
|
||||
normalized = strings.Trim(normalized, "_")
|
||||
if normalized == "" {
|
||||
return "unknown"
|
||||
}
|
||||
return normalized
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package parser
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"mqqt-scrubber/internal/model"
|
||||
)
|
||||
|
||||
type fixtureCase struct {
|
||||
Name string `json:"name"`
|
||||
Topic string `json:"topic"`
|
||||
Payload string `json:"payload"`
|
||||
ReceivedAt string `json:"received_at"`
|
||||
ExpectedMeasurement string `json:"expected_measurement"`
|
||||
ExpectedTimestamp string `json:"expected_timestamp"`
|
||||
ExpectedTags map[string]string `json:"expected_tags"`
|
||||
ExpectedFields map[string]any `json:"expected_fields"`
|
||||
}
|
||||
|
||||
func TestParseTasmotaFixtures(t *testing.T) {
|
||||
contents, err := os.ReadFile("testdata/tasmota_samples.json")
|
||||
if err != nil {
|
||||
t.Fatalf("read fixture file: %v", err)
|
||||
}
|
||||
|
||||
var fixtures []fixtureCase
|
||||
if err := json.Unmarshal(contents, &fixtures); err != nil {
|
||||
t.Fatalf("parse fixture file: %v", err)
|
||||
}
|
||||
|
||||
for _, fixture := range fixtures {
|
||||
t.Run(fixture.Name, func(t *testing.T) {
|
||||
receivedAt, err := time.Parse(time.RFC3339, fixture.ReceivedAt)
|
||||
if err != nil {
|
||||
t.Fatalf("parse receivedAt: %v", err)
|
||||
}
|
||||
|
||||
expectedTimestamp, err := time.Parse(time.RFC3339, fixture.ExpectedTimestamp)
|
||||
if err != nil {
|
||||
t.Fatalf("parse expected timestamp: %v", err)
|
||||
}
|
||||
|
||||
records, err := ParseTasmota(model.RawMessage{
|
||||
Topic: fixture.Topic,
|
||||
Payload: []byte(fixture.Payload),
|
||||
ReceivedAt: receivedAt,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ParseTasmota returned error: %v", err)
|
||||
}
|
||||
|
||||
if len(records) != 1 {
|
||||
t.Fatalf("expected 1 record, got %d", len(records))
|
||||
}
|
||||
|
||||
record := records[0]
|
||||
if record.Measurement != fixture.ExpectedMeasurement {
|
||||
t.Fatalf("unexpected measurement: got %s want %s", record.Measurement, fixture.ExpectedMeasurement)
|
||||
}
|
||||
|
||||
if !record.Timestamp.Equal(expectedTimestamp) {
|
||||
t.Fatalf("unexpected timestamp: got %s want %s", record.Timestamp.Format(time.RFC3339), expectedTimestamp.Format(time.RFC3339))
|
||||
}
|
||||
|
||||
for key, value := range fixture.ExpectedTags {
|
||||
if record.Tags[key] != value {
|
||||
t.Fatalf("unexpected tag %s: got %q want %q", key, record.Tags[key], value)
|
||||
}
|
||||
}
|
||||
|
||||
for key, value := range fixture.ExpectedFields {
|
||||
fieldValue, ok := record.Fields[key]
|
||||
if !ok {
|
||||
t.Fatalf("expected field %s to be present", key)
|
||||
}
|
||||
|
||||
if !fieldEquals(fieldValue, value) {
|
||||
t.Fatalf("unexpected field %s: got %#v want %#v", key, fieldValue, value)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func fieldEquals(got any, want any) bool {
|
||||
switch typedWant := want.(type) {
|
||||
case float64:
|
||||
typedGot, ok := got.(float64)
|
||||
return ok && typedGot == typedWant
|
||||
case string:
|
||||
typedGot, ok := got.(string)
|
||||
return ok && typedGot == typedWant
|
||||
case bool:
|
||||
typedGot, ok := got.(bool)
|
||||
return ok && typedGot == typedWant
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
+148
@@ -0,0 +1,148 @@
|
||||
[
|
||||
{
|
||||
"name": "lwt-online",
|
||||
"topic": "tele/tasmota_896001/LWT",
|
||||
"payload": "Online",
|
||||
"received_at": "2026-03-12T15:21:39Z",
|
||||
"expected_measurement": "tasmota_lwt",
|
||||
"expected_timestamp": "2026-03-12T15:21:39Z",
|
||||
"expected_tags": {
|
||||
"device": "tasmota_896001",
|
||||
"message_type": "lwt",
|
||||
"source": "tasmota"
|
||||
},
|
||||
"expected_fields": {
|
||||
"state": "Online",
|
||||
"online": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "lwt-offline-hyphenated-device",
|
||||
"topic": "tele/tasmota-prusa-mini/LWT",
|
||||
"payload": "Offline",
|
||||
"received_at": "2026-03-12T15:21:39Z",
|
||||
"expected_measurement": "tasmota_lwt",
|
||||
"expected_timestamp": "2026-03-12T15:21:39Z",
|
||||
"expected_tags": {
|
||||
"device": "tasmota_prusa_mini",
|
||||
"message_type": "lwt",
|
||||
"source": "tasmota"
|
||||
},
|
||||
"expected_fields": {
|
||||
"state": "Offline",
|
||||
"online": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "state-single-relay",
|
||||
"topic": "tele/tasmota_67850B/STATE",
|
||||
"payload": "{\"Time\":\"2025-10-28T11:56:55\",\"Uptime\":\"0T15:35:12\",\"UptimeSec\":56112,\"Heap\":27,\"SleepMode\":\"Dynamic\",\"Sleep\":50,\"LoadAvg\":19,\"MqttCount\":1,\"POWER\":\"ON\",\"Wifi\":{\"AP\":1,\"SSId\":\"Home_MiNi_smart\",\"BSSId\":\"02:E2:C6:A9:5F:E9\",\"Channel\":6,\"RSSI\":78,\"Signal\":-61,\"LinkCount\":1,\"Downtime\":\"0T00:00:05\"}}",
|
||||
"received_at": "2025-10-28T11:56:54Z",
|
||||
"expected_measurement": "tasmota_state",
|
||||
"expected_timestamp": "2025-10-28T11:56:55Z",
|
||||
"expected_tags": {
|
||||
"device": "tasmota_67850b",
|
||||
"message_type": "state",
|
||||
"source": "tasmota"
|
||||
},
|
||||
"expected_fields": {
|
||||
"power": "ON",
|
||||
"uptime_sec": 56112,
|
||||
"wifi_rssi": 78,
|
||||
"wifi_signal": -61
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "state-multi-relay",
|
||||
"topic": "tele/tasmota_725D2D/STATE",
|
||||
"payload": "{\"Time\":\"2026-03-12T16:23:14\",\"Uptime\":\"4T07:10:12\",\"UptimeSec\":371412,\"Heap\":24,\"SleepMode\":\"Dynamic\",\"Sleep\":50,\"LoadAvg\":19,\"MqttCount\":1,\"POWER1\":\"OFF\",\"POWER2\":\"OFF\",\"POWER3\":\"OFF\",\"POWER4\":\"OFF\",\"Wifi\":{\"AP\":1,\"SSId\":\"Home_MiNi_smart\",\"BSSId\":\"02:E2:C6:A9:5F:E9\",\"Channel\":6,\"Mode\":\"11n\",\"RSSI\":58,\"Signal\":-71,\"LinkCount\":1,\"Downtime\":\"0T00:00:05\"}}",
|
||||
"received_at": "2026-03-12T16:23:14Z",
|
||||
"expected_measurement": "tasmota_state",
|
||||
"expected_timestamp": "2026-03-12T16:23:14Z",
|
||||
"expected_tags": {
|
||||
"device": "tasmota_725d2d",
|
||||
"message_type": "state",
|
||||
"source": "tasmota"
|
||||
},
|
||||
"expected_fields": {
|
||||
"power1": "OFF",
|
||||
"power4": "OFF",
|
||||
"wifi_mode": "11n",
|
||||
"wifi_rssi": 58
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "state-with-berry",
|
||||
"topic": "tele/tasmota_C8BD20/STATE",
|
||||
"payload": "{\"Time\":\"2026-03-12T16:23:11\",\"Uptime\":\"4T07:10:08\",\"UptimeSec\":371408,\"Heap\":142,\"SleepMode\":\"Dynamic\",\"Sleep\":50,\"LoadAvg\":19,\"MqttCount\":2,\"Berry\":{\"HeapUsed\":4,\"Objects\":46},\"POWER\":\"ON\",\"Wifi\":{\"AP\":1,\"SSId\":\"Home_MiNi_smart\",\"BSSId\":\"32:5A:4C:53:3F:56\",\"Channel\":1,\"Mode\":\"HE20\",\"RSSI\":96,\"Signal\":-52,\"LinkCount\":1,\"Downtime\":\"0T00:00:03\"}}",
|
||||
"received_at": "2026-03-12T16:23:11Z",
|
||||
"expected_measurement": "tasmota_state",
|
||||
"expected_timestamp": "2026-03-12T16:23:11Z",
|
||||
"expected_tags": {
|
||||
"device": "tasmota_c8bd20",
|
||||
"message_type": "state",
|
||||
"source": "tasmota"
|
||||
},
|
||||
"expected_fields": {
|
||||
"berry_heap_used": 4,
|
||||
"berry_objects": 46,
|
||||
"power": "ON",
|
||||
"wifi_mode": "HE20"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "sensor-energy-only",
|
||||
"topic": "tele/tasmota_C88994/SENSOR",
|
||||
"payload": "{\"Time\":\"2026-03-12T16:23:13\",\"ENERGY\":{\"TotalStartTime\":\"2026-02-04T19:13:40\",\"Total\":41.385,\"Yesterday\":1.124,\"Today\":0.799,\"Period\":0,\"Power\":1,\"ApparentPower\":4,\"ReactivePower\":4,\"Factor\":0.22,\"Voltage\":231,\"Current\":0.016}}",
|
||||
"received_at": "2026-03-12T16:23:13Z",
|
||||
"expected_measurement": "tasmota_sensor",
|
||||
"expected_timestamp": "2026-03-12T16:23:13Z",
|
||||
"expected_tags": {
|
||||
"device": "tasmota_c88994",
|
||||
"message_type": "sensor",
|
||||
"source": "tasmota"
|
||||
},
|
||||
"expected_fields": {
|
||||
"energy_power": 1,
|
||||
"energy_voltage": 231,
|
||||
"energy_total": 41.385,
|
||||
"energy_current": 0.016
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "sensor-analog-temperature",
|
||||
"topic": "tele/tasmota_896001/SENSOR",
|
||||
"payload": "{\"Time\":\"2026-03-12T16:25:38\",\"ANALOG\":{\"Temperature\":33.8},\"ENERGY\":{\"TotalStartTime\":\"2022-12-30T00:18:41\",\"Total\":1.413,\"Yesterday\":0.000,\"Today\":0.000,\"Period\":0,\"Power\":0,\"ApparentPower\":0,\"ReactivePower\":0,\"Factor\":0.00,\"Voltage\":0,\"Current\":0.000},\"TempUnit\":\"C\"}",
|
||||
"received_at": "2026-03-12T16:25:38Z",
|
||||
"expected_measurement": "tasmota_sensor",
|
||||
"expected_timestamp": "2026-03-12T16:25:38Z",
|
||||
"expected_tags": {
|
||||
"device": "tasmota_896001",
|
||||
"message_type": "sensor",
|
||||
"source": "tasmota"
|
||||
},
|
||||
"expected_fields": {
|
||||
"analog_temperature": 33.8,
|
||||
"temp_unit": "C",
|
||||
"energy_total": 1.413
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "sensor-analog-a0",
|
||||
"topic": "tele/tasmota_725D2D/SENSOR",
|
||||
"payload": "{\"Time\":\"2026-03-12T16:23:14\",\"ANALOG\":{\"A0\":1024},\"ENERGY\":{\"TotalStartTime\":\"2025-05-23T14:48:03\",\"Total\":15.782,\"Yesterday\":0.000,\"Today\":0.000,\"Period\":0,\"Power\":0,\"ApparentPower\":0,\"ReactivePower\":0,\"Factor\":0.00,\"Voltage\":0,\"Current\":0.000}}",
|
||||
"received_at": "2026-03-12T16:23:14Z",
|
||||
"expected_measurement": "tasmota_sensor",
|
||||
"expected_timestamp": "2026-03-12T16:23:14Z",
|
||||
"expected_tags": {
|
||||
"device": "tasmota_725d2d",
|
||||
"message_type": "sensor",
|
||||
"source": "tasmota"
|
||||
},
|
||||
"expected_fields": {
|
||||
"analog_a0": 1024,
|
||||
"energy_total": 15.782,
|
||||
"energy_power": 0
|
||||
}
|
||||
}
|
||||
]
|
||||
Reference in New Issue
Block a user