current progress state

Author: 0xzer

.gitignore

@@ -5,3 +5,4 @@ logs/
 !example-config.yaml
 !.pre-commit-config.yaml
 /start
+/pkg/mautrix

ROADMAP.md

@@ -1,35 +1,35 @@
 # Features & roadmap
 
 * Matrix → Twitter
-  * [ ] Message content
-    * [ ] Text
+  * [+] Message content
+    * [+] Text
     * [ ] Formatting
-    * [ ] Media
-      * [ ] Images
-      * [ ] Videos
-      * [ ] Gifs
-  * [ ] Message reactions
+    * [+] Media
+      * [+] Images
+      * [+] Videos
+      * [+] Gifs
+  * [+] Message reactions
   * [ ] Typing notifications
   * [ ] Read receipts
 * Twitter → Matrix
-  * [ ] Message content
-    * [ ] Text
+  * [+] Message content
+    * [+] Text
     * [ ] Formatting
-    * [ ] Media
-      * [ ] Images
-      * [ ] Videos
-      * [ ] Gifs
-  * [ ] Message reactions
-  * [ ] Message history
-    * [ ] When creating portal
+    * [+] Media
+      * [+] Images
+      * [+] Videos
+      * [+] Gifs
+  * [+] Message reactions
+  * [+] Message history
+    * [+] When creating portal
     * [ ] Missed messages
-  * [ ] Avatars
+  * [+] Avatars
   * [ ] † Typing notifications
   * [ ] † Read receipts
 * Misc
-  * [ ] Automatic portal creation
-    * [ ] At startup
-    * [ ] When receiving invite or message
+  * [+] Automatic portal creation
+    * [+] At startup
+    * [+] When receiving invite or message
   * [ ] Provisioning API for logging in
   * [ ] Private chat creation by inviting Matrix puppet of Twitter user to new room
   * [ ] Option to use own Matrix account for messages sent from other Twitter clients

build.sh

@@ -1,4 +1,4 @@
 #!/bin/sh
 MAUTRIX_VERSION=$(cat go.mod | grep 'maunium.net/go/mautrix ' | awk '{ print $2 }' | head -n1)
 GO_LDFLAGS="-s -w -X main.Tag=$(git describe --exact-match --tags 2>/dev/null) -X main.Commit=$(git rev-parse HEAD) -X 'main.BuildTime=`date -Iseconds`' -X 'maunium.net/go/mautrix.GoModVersion=$MAUTRIX_VERSION'"
-go build -ldflags="$GO_LDFLAGS" ./cmd/mautrix-twitter "$@"
+go build -ldflags="$GO_LDFLAGS" ./cmd/mautrix-twitter "$@"

cmd/mautrix-twitter/main.go

@@ -1,4 +1,4 @@
-// mautrix-twitter - A Matrix-Slack puppeting bridge.
+// mautrix-twitter - A Matrix-Twitter puppeting bridge.
 // Copyright (C) 2024 Tulir Asokan
 //
 // This program is free software: you can redistribute it and/or modify
@@ -16,5 +16,27 @@
 
 package main
 
+import (
+	"go.mau.fi/mautrix-twitter/pkg/connector"
+	"maunium.net/go/mautrix/bridgev2/matrix/mxmain"
+)
+
+// Information to find out exactly which commit the bridge was built from.
+// These are filled at build time with the -X linker flag.
+var (
+	Tag       = "unknown"
+	Commit    = "unknown"
+	BuildTime = "unknown"
+)
+
 func main() {
+	m := mxmain.BridgeMain{
+		Name:        "mautrix-twitter",
+		URL:         "https://github.com/mautrix/twitter",
+		Description: "A Matrix-Twitter puppeting bridge.",
+		Version:     "0.16.0",
+		Connector: connector.NewConnector(),
+	}
+	m.InitVersion(Tag, Commit, BuildTime)
+	m.Run()
 }

go.mod

@@ -7,25 +7,33 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/mattn/go-colorable v0.1.13
 	github.com/rs/zerolog v1.33.0
-	go.mau.fi/util v0.6.0
+	go.mau.fi/util v0.6.1-0.20240802175451-b430ebbffc98
 	golang.org/x/net v0.27.0
-	maunium.net/go/mautrix v0.19.1-0.20240719130542-cc5f225bc61c
+	maunium.net/go/mautrix v0.19.1-0.20240720173515-24ead553b23b
 )
 
+replace maunium.net/go/mautrix v0.19.1-0.20240720173515-24ead553b23b => ./pkg/mautrix
+
 require (
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/lib/pq v1.10.9 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
-	github.com/tidwall/gjson v1.17.1 // indirect
+	github.com/mattn/go-sqlite3 v1.14.22 // indirect
+	github.com/rs/xid v1.5.0 // indirect
+	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect
+	github.com/tidwall/gjson v1.17.3 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tidwall/sjson v1.2.5 // indirect
 	github.com/yuin/goldmark v1.7.4 // indirect
 	go.mau.fi/zeroconfig v0.1.3 // indirect
 	golang.org/x/crypto v0.25.0 // indirect
-	golang.org/x/exp v0.0.0-20240707233637-46b078467d37 // indirect
+	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
 	golang.org/x/sys v0.22.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	maunium.net/go/mauflag v1.0.0 // indirect
 )

go.sum

@@ -16,22 +16,29 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
+github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
 github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
+github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=
 github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
+github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0=
+github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U=
-github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.17.3 h1:bwWLZU7icoKRG+C+0PNwIKC6FCJO/Q3p2pZvuP0jN94=
+github.com/tidwall/gjson v1.17.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
 github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
@@ -40,27 +47,29 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
 github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
 github.com/yuin/goldmark v1.7.4 h1:BDXOHExt+A7gwPCJgPIIq7ENvceR7we7rOS9TNoLZeg=
 github.com/yuin/goldmark v1.7.4/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
-go.mau.fi/util v0.6.0 h1:W6SyB3Bm/GjenQ5iq8Z8WWdN85Gy2xS6L0wmnR7SVjg=
-go.mau.fi/util v0.6.0/go.mod h1:ljYdq3sPfpICc3zMU+/mHV/sa4z0nKxc67hSBwnrk8U=
+go.mau.fi/util v0.6.1-0.20240802175451-b430ebbffc98 h1:gJ0peWecBm6TtlxKFVIc1KbooXSCHtPfsfb2Eha5A0A=
+go.mau.fi/util v0.6.1-0.20240802175451-b430ebbffc98/go.mod h1:S1juuPWGau2GctPY3FR/4ec/MDLhAG2QPhdnUwpzWIo=
 go.mau.fi/zeroconfig v0.1.3 h1:As9wYDKmktjmNZW5i1vn8zvJlmGKHeVxHVIBMXsm4kM=
 go.mau.fi/zeroconfig v0.1.3/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70=
 golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
 golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
-golang.org/x/exp v0.0.0-20240707233637-46b078467d37 h1:uLDX+AfeFCct3a2C7uIWBKMJIR3CJMhcgfrUAqjRK6w=
-golang.org/x/exp v0.0.0-20240707233637-46b078467d37/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
+golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
+golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
 golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
 golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
 golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
 gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-maunium.net/go/mautrix v0.19.1-0.20240719130542-cc5f225bc61c h1:wjQNGvMn85CRf8bqwqXsg55hvEuxIO8WH0SvOhzv/1E=
-maunium.net/go/mautrix v0.19.1-0.20240719130542-cc5f225bc61c/go.mod h1:UE+mSQ4sDUuJMbjN0aB9EjQSGgXd48AzMvZ6+QJV1k8=
+maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M=
+maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA=

pkg/connector/backfill.go

@@ -0,0 +1,51 @@
+package connector
+
+import (
+	"context"
+	"log"
+	"os"
+
+	"go.mau.fi/mautrix-twitter/pkg/twittermeow/data/payload"
+	"go.mau.fi/mautrix-twitter/pkg/twittermeow/data/types"
+	"maunium.net/go/mautrix/bridgev2"
+	"maunium.net/go/mautrix/bridgev2/networkid"
+)
+
+var _ bridgev2.BackfillingNetworkAPI = (*TwitterClient)(nil)
+
+func (tc *TwitterClient) FetchMessages(ctx context.Context, params bridgev2.FetchMessagesParams) (*bridgev2.FetchMessagesResponse, error) {
+	conversationId := string(params.Portal.PortalKey.ID)
+	cursor := params.Cursor
+    //count := params.Count
+
+	reqQuery := payload.DmRequestQuery{}.Default()
+	reqQuery.Count = 25
+	messageResp, err := tc.client.FetchConversationContext(conversationId, reqQuery, payload.CONTEXT_FETCH_DM_CONVERSATION_HISTORY)
+	if err != nil {
+		return nil, err
+	}
+
+	if cursor != "" {
+		log.Println("found cursor:", params)
+		os.Exit(1)
+	}
+
+	messages, err := messageResp.ConversationTimeline.GetMessageEntriesByConversationID(conversationId, true)
+	if err != nil {
+		return nil, err
+	}
+
+	backfilledMessages, err := tc.MessagesToBackfillMessages(ctx, messages, messageResp.ConversationTimeline.GetConversationByID(conversationId))
+	if err != nil {
+		return nil, err
+	}
+
+	fetchMessagesResp := &bridgev2.FetchMessagesResponse{
+		Messages: backfilledMessages,
+		Cursor: networkid.PaginationCursor(messageResp.ConversationTimeline.MinEntryID),
+		HasMore: messageResp.ConversationTimeline.Status == types.HAS_MORE,
+		Forward: params.Forward,
+	}
+
+	return fetchMessagesResp, nil
+}