commit cb2f83e44b875b9238ddd0709f52217d8f88ff4b
Author: MTRNord <mtrnord1@gmail.com>
Date: Sat, 9 Mar 2024 16:31:09 +0100
Initial commit
Diffstat:
| A | .gitignore | | | 3 | +++ |
| A | datatypes/dataTypes.go | | | 20 | ++++++++++++++++++++ |
| A | edsm/edsm.go | | | 96 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | go.mod | | | 11 | +++++++++++ |
| A | go.sum | | | 13 | +++++++++++++ |
| A | journal/journal.go | | | 164 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | main.go | | | 73 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
7 files changed, 380 insertions(+), 0 deletions(-)
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,2 @@
+edsm_uploader
+latest.txt
+\ No newline at end of file
diff --git a/datatypes/dataTypes.go b/datatypes/dataTypes.go
@@ -0,0 +1,20 @@
+package datatypes
+
+// The journal has some standard fields on each line that are common. Timestamp and Event. The rest of the fields are specific to the event.
+// Example line: `{ "timestamp":"2024-03-09T10:49:40Z", "event":"Fileheader", "part":1, "language":"German/DE", "Odyssey":true, "gameversion":"4.0.0.1801", "build":"r300472/r0 " }`
+type JournalLine struct {
+ Timestamp string `json:"timestamp"`
+ Event string `json:"event"`
+}
+
+// The fileheader is special and we need to parse it each time to make sure we know the right version.
+// Example header: `{ "timestamp":"2024-03-09T10:49:40Z", "event":"Fileheader", "part":1, "language":"German/DE", "Odyssey":true, "gameversion":"4.0.0.1801", "build":"r300472/r0 " }`
+type FileHeader struct {
+ Timestamp string `json:"timestamp"`
+ Event string `json:"event"`
+ Part int `json:"part"`
+ Language string `json:"language"`
+ Odyssey bool `json:"Odyssey"`
+ GameVersion string `json:"gameversion"`
+ Build string `json:"build"`
+}
diff --git a/edsm/edsm.go b/edsm/edsm.go
@@ -0,0 +1,96 @@
+package edsm
+
+import (
+ "bytes"
+ "encoding/json"
+ "io"
+
+ "github.com/MTRNord/edsm_uploader/datatypes"
+ "github.com/hashicorp/go-retryablehttp"
+ "github.com/pkg/errors"
+)
+
+type EDSM struct {
+ commanderName string
+ apiKey string
+ client *retryablehttp.Client
+}
+
+func NewEDSM(commanderName string, apiKey string) *EDSM {
+ retryClient := retryablehttp.NewClient()
+ retryClient.RetryMax = 10
+ return &EDSM{
+ commanderName: commanderName,
+ apiKey: apiKey,
+ client: retryClient,
+ }
+}
+
+type RequestJSON struct {
+ CommanderName string `json:"commanderName"`
+ ApiKey string `json:"apiKey"`
+ Software string `json:"fromSoftware"`
+ SoftwareVersion string `json:"fromSoftwareVersion"`
+ GameVersion string `json:"fromGameVersion"`
+ GameBuild string `json:"fromGameBuild"`
+ Message string `json:"message"`
+}
+
+func (e *EDSM) SendJournalLine(fileHeader *datatypes.FileHeader, journalLine string) error {
+ // Send HTTP Post request to https://www.edsm.net/api-journal-v1
+ // If the response is less then 200 then return an error.
+
+ // Build the JSON request.
+ requestJSON := RequestJSON{
+ CommanderName: e.commanderName,
+ ApiKey: e.apiKey,
+ Software: "github.com/MTRNord/edsm_uploader",
+ SoftwareVersion: "0.1.0",
+ GameVersion: fileHeader.GameVersion,
+ GameBuild: fileHeader.Build,
+ Message: journalLine,
+ }
+ // Convert struct to json
+ b, err := json.Marshal(requestJSON)
+ if err != nil {
+ return errors.WithStack(err)
+ }
+
+ resp, err := e.client.Post("https://www.edsm.net/api-journal-v1", "application/json", bytes.NewBuffer(b))
+ if err != nil {
+ return errors.WithStack(err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode < 200 {
+ // Create error from response
+ bodyBytes, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return errors.WithStack(err)
+ }
+ bodyString := string(bodyBytes)
+ return &EDSMError{
+ Status: resp.StatusCode,
+ Message: bodyString,
+ }
+ } /* else {
+ // Log the response
+ bodyBytes, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return errors.WithStack(err)
+ }
+ bodyString := string(bodyBytes)
+ log.Printf("Response from EDSM: %s", bodyString)
+ } */
+
+ return nil
+}
+
+type EDSMError struct {
+ Status int
+ Message string
+}
+
+func (e *EDSMError) Error() string {
+ return e.Message
+}
diff --git a/go.mod b/go.mod
@@ -0,0 +1,11 @@
+module github.com/MTRNord/edsm_uploader
+
+go 1.21.7
+
+require (
+ github.com/hashicorp/go-retryablehttp v0.7.5
+ github.com/mattn/go-sqlite3 v1.14.22
+ github.com/pkg/errors v0.9.1
+)
+
+require github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
diff --git a/go.sum b/go.sum
@@ -0,0 +1,13 @@
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
+github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
+github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
+github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
+github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M=
+github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
+github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
+github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
diff --git a/journal/journal.go b/journal/journal.go
@@ -0,0 +1,164 @@
+package journal
+
+import (
+ "bufio"
+ "bytes"
+ "encoding/json"
+ "log"
+ "os"
+ "sync"
+ "time"
+
+ "github.com/MTRNord/edsm_uploader/datatypes"
+ "github.com/MTRNord/edsm_uploader/edsm"
+ "github.com/pkg/errors"
+
+ _ "github.com/mattn/go-sqlite3"
+)
+
+type FileHeader struct {
+ // Define the fields of the FileHeader struct here.
+}
+
+type Journal struct {
+ edsm *edsm.EDSM
+ lastDate *time.Time
+ fileHeader *datatypes.FileHeader
+}
+
+func NewJournal(edsm *edsm.EDSM) *Journal {
+ // Read lastDate from latest.txt
+ file, err := os.OpenFile("latest.txt", os.O_RDWR|os.O_CREATE, 0755)
+ if err != nil {
+ log.Printf("Error opening latest.txt: %s", err)
+ }
+ defer file.Close()
+ scanner := bufio.NewScanner(file)
+ var lastDate *time.Time
+ if scanner.Scan() {
+ text := scanner.Text()
+ // Check if the latest.txt is empty.
+ // If it is empty then we need to set the lastDate to nil.
+ if text == "" {
+ lastDate = nil
+ } else {
+ lastDateL, err := time.Parse(time.RFC3339, text)
+ if err != nil {
+ log.Printf("Error parsing latest.txt: %s", err)
+ }
+ lastDate = &lastDateL
+ }
+ }
+
+ return &Journal{
+ // TODO: Parse the last date from the sqlite database.
+ // If there is no last date, then we need to start from the beginning and define it as nil.
+ edsm: edsm,
+ lastDate: lastDate,
+ fileHeader: nil,
+ }
+}
+
+func (j *Journal) ParseJournal(journalPath string) error {
+ log.Printf("Parsing journal file: %s", journalPath)
+ file, err := os.Open(journalPath)
+ if err != nil {
+ return errors.WithStack(err)
+ }
+ defer file.Close()
+
+ scanner := bufio.NewScanner(file)
+ // Parse first line first
+ if scanner.Scan() {
+ err := j.parseLine(scanner.Text())
+ if err != nil {
+ return errors.WithStack(err)
+ }
+ }
+ var wg sync.WaitGroup
+ for scanner.Scan() {
+ wg.Add(1)
+ go func(line string) error {
+ defer wg.Done()
+ err := j.parseLine(line)
+ if err != nil {
+ return errors.WithStack(err)
+ }
+ return nil
+ }(scanner.Text())
+ time.Sleep(1 * time.Millisecond)
+ }
+ if err := scanner.Err(); err != nil {
+ return errors.WithStack(err)
+ }
+ wg.Wait()
+
+ return nil
+}
+
+func (j *Journal) storeLastDate(timestamp time.Time) {
+ if j.lastDate == nil || j.lastDate.Before(timestamp) {
+ j.lastDate = ×tamp
+ file, err := os.OpenFile("latest.txt", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755)
+ if err != nil {
+ log.Printf("Error opening latest.txt: %s", err)
+ }
+ defer file.Close()
+ _, err = file.WriteString(timestamp.Format(time.RFC3339))
+ if err != nil {
+ log.Printf("Error writing to latest.txt: %s", err)
+ }
+ }
+}
+
+// This parses the journal line of the elite dangerous journal.
+// TODO: We need to store where we are in the journal so we can pick up where we left off.
+func (j *Journal) parseLine(line string) error {
+ // Parse json line as a JournalLine.
+ var journalLine datatypes.JournalLine
+ bytesString := []byte(line)
+ bytesString = bytes.Trim(bytesString, "\x00")
+ // Exit early if the line is empty
+ if len(bytesString) == 0 {
+ return nil
+ }
+ err := json.Unmarshal(bytesString, &journalLine)
+ if err != nil {
+ return errors.WithStack(err)
+ }
+
+ if journalLine.Event == "Fileheader" {
+ // Parse the json line as a FileHeader.
+ var fileHeader datatypes.FileHeader
+ json.Unmarshal([]byte(line), &fileHeader)
+
+ // Store the file header.
+ j.fileHeader = &fileHeader
+ }
+
+ // If we have a startdate make sure new lines are newer than the startdate.
+ parsedDate, err := time.Parse(time.RFC3339, journalLine.Timestamp)
+ if err != nil {
+ return errors.WithStack(err)
+ }
+ if j.lastDate != nil {
+ lastDate, err := time.Parse(time.RFC3339, j.lastDate.Format(time.RFC3339))
+ if err != nil {
+ return errors.WithStack(err)
+ }
+ if parsedDate.Before(lastDate) {
+ return nil
+ }
+ }
+
+ // Send the line to edsm.
+ err = j.edsm.SendJournalLine(j.fileHeader, line)
+ if err != nil {
+ return errors.WithStack(err)
+ }
+
+ // Store the last date.
+ j.storeLastDate(parsedDate)
+
+ return nil
+}
diff --git a/main.go b/main.go
@@ -0,0 +1,73 @@
+package main
+
+import (
+ "os"
+ "path/filepath"
+ "sort"
+ "strings"
+ "time"
+
+ "log"
+
+ "github.com/MTRNord/edsm_uploader/edsm"
+ "github.com/MTRNord/edsm_uploader/journal"
+ "github.com/pkg/errors"
+)
+
+func main() {
+ // Take as a first positional argument the path to the folder where the journal files are located.
+ // Take as a second positional argument the commander name.
+ // Take as a third positional argument the EDSM API key.
+
+ jounnalPath := os.Args[1]
+ commanderName := os.Args[2]
+ apiKey := os.Args[3]
+
+ // Create a new EDSM object.
+ edsm := edsm.NewEDSM(commanderName, apiKey)
+
+ // Find all the journal files in the folder and parse them after sorting them by date.
+ files := make(map[string]string)
+ err := filepath.WalkDir(jounnalPath, func(path string, d os.DirEntry, err error) error {
+ if err != nil {
+ return errors.WithStack(err)
+ }
+ if !d.IsDir() {
+ // If the file starts with "Journal." then we add it to the files map.
+ // The date is after "Journal." and before the first ".".
+ // Example: "Journal.2024-03-09T104947.01.log"
+ // The date is "2024-03-09T104947.01"
+ if d.Name()[:8] == "Journal." {
+ name := d.Name()
+ parts := strings.Split(name, ".")
+ date := parts[1]
+ files[date] = path
+ }
+
+ }
+ return nil
+ })
+ if err != nil {
+ log.Fatalf("Error walking the path: %+v", err)
+ }
+
+ // Sort the files by date.
+ keys := make([]string, 0, len(files))
+ for k := range files {
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
+
+ journal_obj := journal.NewJournal(edsm)
+ for _, k := range keys {
+ // Parse the journal file.
+ err := journal_obj.ParseJournal(files[k])
+ if err != nil {
+ log.Fatalf("Error parsing journal file: %+v", err)
+ }
+ // sleep for 1 second to not overload the EDSM API.
+ time.Sleep(1 * time.Second)
+ }
+
+ log.Println("Done parsing journal files.")
+}
