mirror of
https://github.com/firestar5683/StarPilot.git
synced 2026-11-08 13:33:42 +08:00
firestar5683
ab81464bd1
Add navigation and map matching providers with the vendored map service sources. Co-authored-by: Daniel Koepping <elkoled@gmail.com> Co-authored-by: Danny <dannyirwin31@yahoo.com> Co-authored-by: Prabhaav Pillai <prabhaav@msu.edu> Co-authored-by: dirwin31 <83434411+dirwin31@users.noreply.github.com>
411 lines
14 KiB
Go
411 lines
14 KiB
Go
package maps
|
|
|
|
import (
|
|
"archive/tar"
|
|
"bytes"
|
|
"compress/gzip"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/binary"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"syscall"
|
|
"testing"
|
|
"time"
|
|
|
|
"capnproto.org/go/capnp/v3"
|
|
"google.golang.org/protobuf/encoding/protowire"
|
|
"pfeifer.dev/mapd/cereal/offline"
|
|
m "pfeifer.dev/mapd/math"
|
|
)
|
|
|
|
func pbfBytesField(dst []byte, field protowire.Number, data []byte) []byte {
|
|
dst = protowire.AppendTag(dst, field, protowire.BytesType)
|
|
return protowire.AppendBytes(dst, data)
|
|
}
|
|
|
|
func archiveTiles(t *testing.T, count int) []byte {
|
|
t.Helper()
|
|
var out bytes.Buffer
|
|
gz := gzip.NewWriter(&out)
|
|
tw := tar.NewWriter(gz)
|
|
expected, err := expectedSnapshotTiles([4]float64{0, 0, 2, 2})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, rel := range expected[:count] {
|
|
var a, b, c, d float64
|
|
parts := strings.Split(filepath.Base(rel), "_")
|
|
if len(parts) != 4 {
|
|
t.Fatal(rel)
|
|
}
|
|
if _, err := fmt.Sscanf(strings.Join(parts, " "), "%f %f %f %f", &a, &b, &c, &d); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
msg, seg, err := capnp.NewMessage(capnp.SingleSegment(nil))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
tile, err := offline.NewRootOffline(seg)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
tile.SetMinLat(a)
|
|
tile.SetMinLon(b)
|
|
tile.SetMaxLat(c)
|
|
tile.SetMaxLon(d)
|
|
tile.SetOverlap(.001)
|
|
if _, err := tile.NewWays(0); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
data, err := msg.MarshalPacked()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
name := "offline/" + rel
|
|
if err := tw.WriteHeader(&tar.Header{Name: name, Typeflag: tar.TypeReg, Mode: 0644, Size: int64(len(data))}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := tw.Write(data); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := tw.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := gz.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return out.Bytes()
|
|
}
|
|
|
|
func TestSnapshotArchiveAtomicSelectionAndMalformedInputs(t *testing.T) {
|
|
root := t.TempDir()
|
|
pbf := filepath.Join("..", "testdata", "synthetic_snapshot.osm.pbf")
|
|
oldID, err := AdmitSnapshot(context.Background(), AdmitOptions{Root: root, Bounds: [4]float64{35, -98, 35.25, -97.75}, Overlap: .001, PBF: pbf, SourceRevision: "test", UpstreamRevision: "test", SourceDigest: "test"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
oldDir, oldReceipt, err := ResolveSnapshot(root)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
oldLoader := SnapshotLoader(oldDir, oldReceipt)
|
|
archiveDir := filepath.Join(t.TempDir(), "archives")
|
|
if err := os.MkdirAll(filepath.Join(archiveDir, "0"), 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
archive := filepath.Join(archiveDir, "0", "0.tar.gz")
|
|
options := AdmitOptions{Root: root, Bounds: [4]float64{0, 0, 2, 2}, Overlap: .001, ArchiveDir: archiveDir, SourceRevision: "test", UpstreamRevision: "test", SourceDigest: "test"}
|
|
if err := os.WriteFile(archive, archiveTiles(t, 63), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := AdmitSnapshot(context.Background(), options); err == nil {
|
|
t.Fatal("missing cell admitted")
|
|
}
|
|
selected, _, err := ResolveSnapshot(root)
|
|
if err != nil || filepath.Base(selected) != oldID {
|
|
t.Fatalf("partial archive changed selector: %v", err)
|
|
}
|
|
if err := os.WriteFile(archive, archiveTiles(t, 64), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
canceled, stop := context.WithCancel(context.Background())
|
|
stop()
|
|
if _, err := AdmitSnapshot(canceled, options); err == nil {
|
|
t.Fatal("canceled admission selected a generation")
|
|
}
|
|
newID, err := AdmitSnapshot(context.Background(), options)
|
|
if err != nil || newID == oldID {
|
|
t.Fatalf("complete archive rejected: %v", err)
|
|
}
|
|
selected, receipt, err := ResolveSnapshot(root)
|
|
if err != nil || filepath.Base(selected) != newID || len(receipt.Tiles) != 64 {
|
|
t.Fatalf("archive receipt wrong: %v", err)
|
|
}
|
|
if _, err := SnapshotLoader(selected, receipt)(m.NewPosition(.125, .125)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if oldWay, err := oldLoader(m.NewPosition(35.125, -97.875)); err != nil || !oldWay.Loaded || oldWay.Ways.Len() != 1 {
|
|
t.Fatalf("selector changed an already pinned process generation: %v", err)
|
|
}
|
|
if _, err := SnapshotLoader(filepath.Join(root, "generations", oldID), receipt)(m.NewPosition(.125, .125)); err == nil {
|
|
t.Fatal("different generation accepted as cache identity")
|
|
}
|
|
}
|
|
|
|
func TestSnapshotSpecialFilesAndOversizeAreRejected(t *testing.T) {
|
|
root := t.TempDir()
|
|
pbf := filepath.Join("..", "testdata", "synthetic_snapshot.osm.pbf")
|
|
id, err := AdmitSnapshot(context.Background(), AdmitOptions{Root: root, Bounds: [4]float64{35, -98, 35.25, -97.75}, Overlap: .001, PBF: pbf, SourceRevision: "test", UpstreamRevision: "test", SourceDigest: "test"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
selected, receipt, err := ResolveSnapshot(root)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
tile := filepath.Join(selected, receipt.Tiles[0].Path)
|
|
original, err := os.ReadFile(tile)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, kind := range []string{"fifo", "symlink", "oversize"} {
|
|
t.Run(kind, func(t *testing.T) {
|
|
if err := os.Remove(tile); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
switch kind {
|
|
case "fifo":
|
|
if err := syscall.Mkfifo(tile, 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
case "symlink":
|
|
if err := os.Symlink(filepath.Join(root, "current.json"), tile); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
case "oversize":
|
|
f, err := os.Create(tile)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.Truncate(maxOfflinePackedBytes + 1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.Close()
|
|
}
|
|
if _, err := SnapshotLoader(selected, receipt)(m.NewPosition(35.125, -97.875)); err == nil {
|
|
t.Fatal("malformed requested tile accepted")
|
|
}
|
|
if err := os.Remove(tile); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(tile, original, 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
})
|
|
}
|
|
selector := filepath.Join(root, "current.json")
|
|
savedSelector, err := os.ReadFile(selector)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.Remove(selector); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := syscall.Mkfifo(selector, 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, _, err := ResolveSnapshot(root); err == nil {
|
|
t.Fatal("FIFO selector accepted")
|
|
}
|
|
os.Remove(selector)
|
|
os.WriteFile(selector, savedSelector, 0600)
|
|
os.Remove(selector)
|
|
if err := os.Symlink(filepath.Join(root, "generations", id, "receipt.json"), selector); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, _, err := ResolveSnapshot(root); err == nil {
|
|
t.Fatal("symlink selector accepted")
|
|
}
|
|
os.Remove(selector)
|
|
f, err := os.Create(selector)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.Truncate(maxSelectorBytes + 1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.Close()
|
|
if _, _, err := ResolveSnapshot(root); err == nil {
|
|
t.Fatal("oversize selector accepted")
|
|
}
|
|
os.Remove(selector)
|
|
os.WriteFile(selector, savedSelector, 0600)
|
|
receiptPath := filepath.Join(root, "generations", id, "receipt.json")
|
|
savedReceipt, err := os.ReadFile(receiptPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
os.Remove(receiptPath)
|
|
syscall.Mkfifo(receiptPath, 0600)
|
|
if _, _, err := ResolveSnapshot(root); err == nil {
|
|
t.Fatal("FIFO receipt accepted")
|
|
}
|
|
os.Remove(receiptPath)
|
|
os.WriteFile(receiptPath, savedReceipt, 0600)
|
|
os.Remove(receiptPath)
|
|
if err := os.Symlink(selector, receiptPath); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, _, err := ResolveSnapshot(root); err == nil {
|
|
t.Fatal("symlink receipt accepted")
|
|
}
|
|
os.Remove(receiptPath)
|
|
f, err = os.Create(receiptPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := f.Truncate(maxReceiptBytes + 1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.Close()
|
|
if _, _, err := ResolveSnapshot(root); err == nil {
|
|
t.Fatal("oversize receipt accepted")
|
|
}
|
|
os.Remove(receiptPath)
|
|
os.WriteFile(receiptPath, savedReceipt, 0600)
|
|
}
|
|
|
|
func TestMalformedCanonicalReceiptBoundsRejectBeforeArchiveLoops(t *testing.T) {
|
|
root := t.TempDir()
|
|
receipt := SnapshotReceipt{Version: 1, SourceKind: "provided-group-archives", GeneratedUnix: 1800000000,
|
|
Bounds: [4]float64{-1e300, 0, 1e300, 0}, Overlap: .001, SourceRevision: "test", UpstreamRevision: "test", SourceDigest: "test",
|
|
TileSchema: "offline.capnp:0xda3a0d9284ca402f", Attribution: attribution,
|
|
Inputs: []SnapshotInput{{Name: "0/0.tar.gz", Size: 1, SHA256: strings.Repeat("a", 64)}},
|
|
Tiles: []SnapshotTile{{Path: "0/0/tile", Size: 1, SHA256: strings.Repeat("b", 64)}}}
|
|
raw, err := json.Marshal(receipt)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
hash := sha256.Sum256(raw)
|
|
id := hex.EncodeToString(hash[:])
|
|
dir := filepath.Join(root, "generations", id)
|
|
if err := os.MkdirAll(dir, 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(dir, "receipt.json"), raw, 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
selector, _ := json.Marshal(snapshotSelector{Version: 1, Generation: id})
|
|
if err := os.WriteFile(filepath.Join(root, "current.json"), selector, 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, _, err := ResolveSnapshot(root); err == nil {
|
|
t.Fatal("out-of-range canonical receipt accepted")
|
|
}
|
|
}
|
|
func pbfVarintField(dst []byte, field protowire.Number, value uint64) []byte {
|
|
dst = protowire.AppendTag(dst, field, protowire.VarintType)
|
|
return protowire.AppendVarint(dst, value)
|
|
}
|
|
func pbfPacked(values ...uint64) []byte {
|
|
var data []byte
|
|
for _, value := range values {
|
|
data = protowire.AppendVarint(data, value)
|
|
}
|
|
return data
|
|
}
|
|
func pbfBlock(kind string, raw []byte) []byte {
|
|
blob := pbfBytesField(nil, 1, raw)
|
|
header := pbfBytesField(nil, 1, []byte(kind))
|
|
header = pbfVarintField(header, 3, uint64(len(blob)))
|
|
data := make([]byte, 4)
|
|
binary.BigEndian.PutUint32(data, uint32(len(header)))
|
|
data = append(data, header...)
|
|
return append(data, blob...)
|
|
}
|
|
func syntheticPBF() []byte {
|
|
header := pbfBytesField(nil, 4, []byte("OsmSchema-V0.6"))
|
|
header = pbfBytesField(header, 5, []byte("LocationsOnWays"))
|
|
var table []byte
|
|
for _, value := range []string{"", "highway", "residential", "maxspeed", "30 mph"} {
|
|
table = pbfBytesField(table, 1, []byte(value))
|
|
}
|
|
var way []byte
|
|
way = pbfVarintField(way, 1, 42)
|
|
way = pbfBytesField(way, 2, pbfPacked(1, 3))
|
|
way = pbfBytesField(way, 3, pbfPacked(2, 4))
|
|
way = pbfBytesField(way, 8, pbfPacked(protowire.EncodeZigZag(1), protowire.EncodeZigZag(1)))
|
|
way = pbfBytesField(way, 9, pbfPacked(protowire.EncodeZigZag(351000000), protowire.EncodeZigZag(1000000)))
|
|
way = pbfBytesField(way, 10, pbfPacked(protowire.EncodeZigZag(-979000000), protowire.EncodeZigZag(0)))
|
|
group := pbfBytesField(nil, 3, way)
|
|
block := pbfBytesField(nil, 1, table)
|
|
block = pbfBytesField(block, 2, group)
|
|
block = pbfVarintField(block, 17, 100)
|
|
return append(pbfBlock("OSMHeader", header), pbfBlock("OSMData", block)...)
|
|
}
|
|
|
|
func TestActualGeneratorAndSnapshotAdmission(t *testing.T) {
|
|
root := t.TempDir()
|
|
checkedIn, err := os.ReadFile(filepath.Join("..", "testdata", "synthetic_snapshot.osm.pbf"))
|
|
if err != nil || !bytes.Equal(checkedIn, syntheticPBF()) {
|
|
t.Fatalf("synthetic PBF fixture drift: %v", err)
|
|
}
|
|
pbf := filepath.Join(t.TempDir(), "synthetic.osm.pbf")
|
|
if err := os.WriteFile(pbf, syntheticPBF(), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
bounds := [4]float64{35, -98, 35.25, -97.5}
|
|
generated := filepath.Join(t.TempDir(), "generated")
|
|
s := OfflineSettings{Box: m.Box{MinPos: m.NewPosition(35, -98), MaxPos: m.NewPosition(35.25, -97.5)},
|
|
Overlap: .001, InputFile: pbf, OutputDirectory: generated, GenerateEmptyFiles: true}
|
|
if err := GenerateOffline(s); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
one, err := FindWaysAroundPositionIn(m.NewPosition(35.125, -97.875), generated)
|
|
if err != nil || !one.Loaded || one.Ways.Len() != 1 {
|
|
t.Fatalf("actual PBF way not loaded: %v %v", err, one.Ways.Len())
|
|
}
|
|
empty, err := FindWaysAroundPositionIn(m.NewPosition(35.125, -97.625), generated)
|
|
if err != nil || !empty.Loaded || empty.Ways.Len() != 0 {
|
|
t.Fatalf("actual empty coverage not loaded: %v %v", err, empty.Ways.Len())
|
|
}
|
|
noEmpty := filepath.Join(t.TempDir(), "no-empty")
|
|
s.OutputDirectory, s.GenerateEmptyFiles = noEmpty, false
|
|
if err := GenerateOffline(s); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := os.Stat(GenerateBoundsFileName(Area{Box: m.Box{MinPos: m.NewPosition(35, -97.75), MaxPos: m.NewPosition(35.25, -97.5)}}, OfflineSettings{OutputDirectory: noEmpty})); !os.IsNotExist(err) {
|
|
t.Fatalf("generator emitted empty cell with flag off: %v", err)
|
|
}
|
|
options := AdmitOptions{Root: root, Bounds: bounds, Overlap: .001, PBF: pbf,
|
|
SourceRevision: "test", UpstreamRevision: "test", SourceDigest: "test", Now: func() time.Time { return time.Unix(1800000000, 0) }}
|
|
invalidOverlap := options
|
|
invalidOverlap.Overlap = .25
|
|
if _, err := AdmitSnapshot(context.Background(), invalidOverlap); err == nil {
|
|
t.Fatal("one-cell overlap accepted")
|
|
}
|
|
id, err := AdmitSnapshot(context.Background(), options)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
selected, receipt, err := ResolveSnapshot(root)
|
|
if err != nil || filepath.Base(selected) != id || len(receipt.Tiles) != 2 || receipt.Tiles[0].Empty || !receipt.Tiles[1].Empty {
|
|
t.Fatalf("bad selected generation: %q %+v %v", selected, receipt.Tiles, err)
|
|
}
|
|
selector, err := os.ReadFile(filepath.Join(root, "current.json"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(selector) == 0 {
|
|
t.Fatal("empty selector")
|
|
}
|
|
receiptBytes, err := os.ReadFile(filepath.Join(selected, "receipt.json"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
digest := sha256.Sum256(receiptBytes)
|
|
if hex.EncodeToString(digest[:]) != id {
|
|
t.Fatal("receipt id differs from exact bytes")
|
|
}
|
|
tile := filepath.Join(selected, receipt.Tiles[0].Path)
|
|
if err := os.WriteFile(tile, []byte("changed"), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
selected, receipt, err = ResolveSnapshot(root)
|
|
if err != nil {
|
|
t.Fatalf("metadata should remain readable: %v", err)
|
|
}
|
|
if _, err := SnapshotLoader(selected, receipt)(m.NewPosition(35.125, -97.875)); err == nil {
|
|
t.Fatal("changed requested tile accepted by pinned loader")
|
|
}
|
|
if _, err := AdmitSnapshot(context.Background(), options); err == nil {
|
|
t.Fatal("existing corrupted generation reselected")
|
|
}
|
|
}
|