Add initial MQTT scrubber service scaffold

2026-03-12 18:12:16 +01:00
parent 957b2c41b3
commit 464f4c3ec4
22 changed files with 4150 additions and 1 deletions
@@ -0,0 +1,184 @@
+package influx
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"mqqt-scrubber/internal/config"
+	"mqqt-scrubber/internal/model"
+)
+
+type Client struct {
+	baseURL    string
+	database   string
+	token      string
+	precision  string
+	httpClient *http.Client
+}
+
+func NewClient(cfg config.InfluxConfig) *Client {
+	return &Client{
+		baseURL:   strings.TrimRight(cfg.URL, "/"),
+		database:  cfg.Database,
+		token:     cfg.Token,
+		precision: cfg.Precision,
+		httpClient: &http.Client{
+			Timeout: 30 * time.Second,
+		},
+	}
+}
+
+func (client *Client) Write(ctx context.Context, records []model.Record) error {
+	if len(records) == 0 {
+		return nil
+	}
+
+	lines := make([]string, 0, len(records))
+	for _, record := range records {
+		line, err := toLineProtocol(record)
+		if err != nil {
+			return err
+		}
+		lines = append(lines, line)
+	}
+
+	writeURL, err := url.Parse(client.baseURL)
+	if err != nil {
+		return fmt.Errorf("parse influx url: %w", err)
+	}
+
+	writeURL.Path = strings.TrimRight(writeURL.Path, "/") + "/api/v3/write_lp"
+	query := writeURL.Query()
+	query.Set("db", client.database)
+	query.Set("precision", client.precision)
+	writeURL.RawQuery = query.Encode()
+
+	body := strings.Join(lines, "\n")
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, writeURL.String(), bytes.NewBufferString(body))
+	if err != nil {
+		return fmt.Errorf("create write request: %w", err)
+	}
+
+	req.Header.Set("Content-Type", "text/plain; charset=utf-8")
+	if client.token != "" {
+		req.Header.Set("Authorization", "Bearer "+client.token)
+	}
+
+	resp, err := client.httpClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("execute write request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode >= http.StatusBadRequest {
+		responseBody, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
+		return fmt.Errorf("influx write failed with status %d: %s", resp.StatusCode, strings.TrimSpace(string(responseBody)))
+	}
+
+	return nil
+}
+
+func toLineProtocol(record model.Record) (string, error) {
+	if record.Measurement == "" {
+		return "", fmt.Errorf("record measurement is required")
+	}
+	if len(record.Fields) == 0 {
+		return "", fmt.Errorf("record fields are required")
+	}
+
+	tagKeys := sortedKeys(record.Tags)
+	fieldKeys := sortedKeysAny(record.Fields)
+
+	var builder strings.Builder
+	builder.WriteString(escapeMeasurement(record.Measurement))
+
+	for _, key := range tagKeys {
+		builder.WriteByte(',')
+		builder.WriteString(escapeTagOrKey(key))
+		builder.WriteByte('=')
+		builder.WriteString(escapeTagOrKey(record.Tags[key]))
+	}
+
+	builder.WriteByte(' ')
+	for index, key := range fieldKeys {
+		if index > 0 {
+			builder.WriteByte(',')
+		}
+
+		builder.WriteString(escapeTagOrKey(key))
+		builder.WriteByte('=')
+
+		formatted, err := formatFieldValue(record.Fields[key])
+		if err != nil {
+			return "", fmt.Errorf("format field %q: %w", key, err)
+		}
+		builder.WriteString(formatted)
+	}
+
+	builder.WriteByte(' ')
+	builder.WriteString(strconv.FormatInt(record.Timestamp.UnixNano(), 10))
+
+	return builder.String(), nil
+}
+
+func sortedKeys(values map[string]string) []string {
+	keys := make([]string, 0, len(values))
+	for key := range values {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	return keys
+}
+
+func sortedKeysAny(values map[string]any) []string {
+	keys := make([]string, 0, len(values))
+	for key := range values {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	return keys
+}
+
+func formatFieldValue(value any) (string, error) {
+	switch typed := value.(type) {
+	case string:
+		escaped := strings.ReplaceAll(typed, `\`, `\\`)
+		escaped = strings.ReplaceAll(escaped, `"`, `\"`)
+		return `"` + escaped + `"`, nil
+	case bool:
+		return strconv.FormatBool(typed), nil
+	case int:
+		return strconv.Itoa(typed) + "i", nil
+	case int64:
+		return strconv.FormatInt(typed, 10) + "i", nil
+	case float64:
+		return strconv.FormatFloat(typed, 'f', -1, 64), nil
+	case float32:
+		return strconv.FormatFloat(float64(typed), 'f', -1, 32), nil
+	default:
+		return "", fmt.Errorf("unsupported field type %T", value)
+	}
+}
+
+func escapeMeasurement(value string) string {
+	return strings.NewReplacer(
+		",", `\,`,
+		" ", `\ `,
+	).Replace(value)
+}
+
+func escapeTagOrKey(value string) string {
+	return strings.NewReplacer(
+		",", `\,`,
+		"=", `\=`,
+		" ", `\ `,
+	).Replace(value)
+}
@@ -0,0 +1,60 @@
+package influx
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"mqqt-scrubber/internal/model"
+	"mqqt-scrubber/internal/parser"
+)
+
+func TestToLineProtocolFromParsedLWT(t *testing.T) {
+	receivedAt := time.Date(2026, time.March, 12, 15, 21, 39, 0, time.UTC)
+	records, err := parser.ParseTasmota(model.RawMessage{
+		Topic:      "tele/tasmota_896001/LWT",
+		Payload:    []byte("Online"),
+		ReceivedAt: receivedAt,
+	})
+	if err != nil {
+		t.Fatalf("ParseTasmota returned error: %v", err)
+	}
+
+	line, err := toLineProtocol(records[0])
+	if err != nil {
+		t.Fatalf("toLineProtocol returned error: %v", err)
+	}
+
+	expected := fmt.Sprintf(
+		"tasmota_lwt,device=tasmota_896001,message_type=lwt,source=tasmota online=true,state=\"Online\" %d",
+		receivedAt.UnixNano(),
+	)
+	if line != expected {
+		t.Fatalf("unexpected line protocol:\n got: %s\nwant: %s", line, expected)
+	}
+}
+
+func TestToLineProtocolFromParsedSensor(t *testing.T) {
+	receivedAt := time.Date(2026, time.March, 12, 16, 23, 13, 0, time.UTC)
+	records, err := parser.ParseTasmota(model.RawMessage{
+		Topic:      "tele/tasmota_C88994/SENSOR",
+		Payload:    []byte(`{"Time":"2026-03-12T16:23:13","ENERGY":{"TotalStartTime":"2026-02-04T19:13:40","Total":41.385,"Yesterday":1.124,"Today":0.799,"Period":0,"Power":1,"ApparentPower":4,"ReactivePower":4,"Factor":0.22,"Voltage":231,"Current":0.016}}`),
+		ReceivedAt: receivedAt,
+	})
+	if err != nil {
+		t.Fatalf("ParseTasmota returned error: %v", err)
+	}
+
+	line, err := toLineProtocol(records[0])
+	if err != nil {
+		t.Fatalf("toLineProtocol returned error: %v", err)
+	}
+
+	expected := fmt.Sprintf(
+		"tasmota_sensor,device=tasmota_c88994,message_type=sensor,source=tasmota energy_apparent_power=4,energy_current=0.016,energy_factor=0.22,energy_period=0,energy_power=1,energy_reactive_power=4,energy_today=0.799,energy_total=41.385,energy_total_start_time=\"2026-02-04T19:13:40\",energy_voltage=231,energy_yesterday=1.124 %d",
+		receivedAt.UnixNano(),
+	)
+	if line != expected {
+		t.Fatalf("unexpected line protocol:\n got: %s\nwant: %s", line, expected)
+	}
+}