added Nostr feed support (#471)

* Add nostr library dependency.

* Add a Nostr URI parsing function.

* Add a function for getting author profile information. Rename nreq() to nreqProfile() to match function's use. Add default relays, and Nostr Client.

* Use a hack for being able to use Nostr URIs while keeping the URL dependency intact.

* Merge nreqProfile into getProfileMetadata. Use default relays if user does not have one.

* Complete the compatibility 'hack' for Nostr URIs, by stripping the bolted-on URL.

* Add function for finding where the author publishes to(the Nostr publish relays).

* Introduce fetchArticlesForAuthor(). Only use getUserPublishRelays() if relays are not found from parsing the Nostr URI.

* Introduce fetchArticlesForAuthor() and AuthorNostrData. Only use getUserPublishRelays() if relays are not found from parsing the Nostr URI.

* Add function to fetch Nostr feed and convert it to Feeder internal models. Actually use AuthorNostrData.

* Wire up everything in the SearchFeedViewModel. Adapt it to take into account the input being handled.

* Check for a Nostr URI first(so as to make the integration work correctly). Fix the if-else in searchForFeeds().

* Add Nostr URI checks when creating/editing the feed, to make sure a Nostr feed can be saved.

* Add Nostr URI checks when creating/editing the feed, to make sure a Nostr feed can be saved, pt. 2.

* Re-use code for checking adapted Nostr URIs.

* Make sure RssLocalSync looks for Nostr feeds when using Nostr-based input.

* Reduce timeouts for fetching from relays.

* Fix incorrect article timestamp issue.

* Retry with default relays if the ones found don't work. Save only the working relays as well.

* Add guardrails so as not to use too many relays when fetching for articles. Add a default set of relays for articles.

* Improve search times by just using the Nostr profile found(may be changed).

* OPML support for Nostr(file imports).

* Update Nostr feed URL by default if it conflicts with one already present in the app.

* Use default relays when guardrails fail, or when user doesn't have them. Clear all relays for user after fetch.

* Adjust Nostr URI validity checks, and improve Nostr support for singular links.

* Add support for parsing and displaying Markdown content.

* Make linter happy.

* Make linter happy, part 2.

* Gradle catalog for rust-nostr dependency.

* Try PR CI again.

* Revert 'Try PR CI again.'

* Try CI again.

* Modify ProGuard rules.

* Disable dependency metadata when building APK/AAB.

* Revert 'Disable dependency metadata when building APK/AAB.'

* Remove relays for which we could not find articles from, before using the default article fetch relays.

* Add release variant as fallback for resolving library variant matching issues. Fix proguard.

* Disable FullTextByDefault option for Nostr feeds, as the full article is fetched anyways.

* Try to fix and optimize the fetching code.

* Try to fix and optimize the fetching code, part 2.

* Revert 'Try to fix and optimize the fetching code, part 2.'

* Make linter happy(again).

* Move markdown dependency to Version catalog.

* Rename some strings in string resources.

* Apply suggested fixes.

* Revert string resource renaming.

* Use getApplication() instead of using DI, to avoid compiler conflicts.

* Bump minimum relays up by 1. Try to make sure we find the author metadata if we dont succeed with the tried relays.

* Apply other suggestions.

app/build.gradle.kts

@@ -99,6 +99,7 @@ android {
                 getDefaultProguardFile("proguard-android-optimize.txt"),
                 "proguard-rules.pro",
             )
+            matchingFallbacks.add("release")
             if (project.hasProperty("STORE_FILE")) {
                 signingConfig = signingConfigs.getByName("release")
             }
@@ -107,7 +108,6 @@ android {
 //            }
         }
     }
-
     testOptions {
         unitTests {
             isReturnDefaultValues = true
@@ -225,6 +225,12 @@ dependencies {
     implementation(libs.openai.client)
     implementation(libs.ktor.client.okhttp)
 
+    // Nostr
+    implementation(libs.rust.nostr)
+
+    // Markdown
+    implementation(libs.jetbrains.markdown)
+
     // Only for debug
     debugImplementation("com.squareup.leakcanary:leakcanary-android:3.0-alpha-1")
 

app/proguard-rules.pro

@@ -37,6 +37,14 @@
 # For Jsoup
 -keep class org.jsoup.**  { *; }
 
+# For Nostr
+-keep class com.sun.jna.** { *; }
+-keep class * implements com.sun.jna.** { *; }
+-dontwarn java.awt.Component
+-dontwarn java.awt.GraphicsEnvironment
+-dontwarn java.awt.HeadlessException
+-dontwarn java.awt.Window
+
 # For Kodein
 -keep, allowobfuscation, allowoptimization class org.kodein.type.TypeReference
 -keep, allowobfuscation, allowoptimization class org.kodein.type.JVMAbstractTypeToken$Companion$WrappingTest

app/src/main/java/com/nononsenseapps/feeder/model/FeedParser.kt

@@ -10,6 +10,7 @@ import com.nononsenseapps.feeder.model.gofeed.GoFeedAdapter
 import com.nononsenseapps.feeder.model.gofeed.GoPerson
 import com.nononsenseapps.feeder.util.Either
 import com.nononsenseapps.feeder.util.flatMap
+import com.nononsenseapps.feeder.util.logDebug
 import com.nononsenseapps.feeder.util.relativeLinkIntoAbsolute
 import com.nononsenseapps.feeder.util.relativeLinkIntoAbsoluteOrThrow
 import com.nononsenseapps.feeder.util.sloppyLinkToStrictURLOrNull
@@ -23,16 +24,39 @@ import okhttp3.OkHttpClient
 import okhttp3.Request
 import okhttp3.Response
 import okhttp3.ResponseBody
+import org.intellij.markdown.flavours.commonmark.CommonMarkFlavourDescriptor
+import org.intellij.markdown.html.HtmlGenerator
+import org.intellij.markdown.parser.MarkdownParser
 import org.jsoup.Jsoup
 import org.jsoup.nodes.Document
 import org.kodein.di.DI
 import org.kodein.di.DIAware
 import org.kodein.di.instance
+import rust.nostr.sdk.Alphabet
+import rust.nostr.sdk.Client
+import rust.nostr.sdk.Coordinate
+import rust.nostr.sdk.Event
+import rust.nostr.sdk.Filter
+import rust.nostr.sdk.Kind
+import rust.nostr.sdk.KindEnum
+import rust.nostr.sdk.Nip19Profile
+import rust.nostr.sdk.Nip21
+import rust.nostr.sdk.Nip21Enum
+import rust.nostr.sdk.NostrSdkException
+import rust.nostr.sdk.PublicKey
+import rust.nostr.sdk.RelayMetadata
+import rust.nostr.sdk.SingleLetterTag
+import rust.nostr.sdk.TagKind
+import rust.nostr.sdk.extractRelayList
+import rust.nostr.sdk.getNip05Profile
 import java.io.IOException
 import java.lang.NullPointerException
 import java.net.MalformedURLException
 import java.net.URL
 import java.net.URLDecoder
+import java.time.Duration
+import java.time.Instant
+import java.time.ZoneId
 import java.util.Locale
 
 private const val YOUTUBE_CHANNEL_ID_ATTR = "data-channel-external-id"
@@ -41,6 +65,14 @@ class FeedParser(override val di: DI) : DIAware {
     private val client: OkHttpClient by instance()
     private val goFeedAdapter = GoFeedAdapter()
 
+    // Initializing the Nostr Client
+    private val nostrClient = Client()
+
+    // The default relays to get info from, separated by purpose.
+    private val defaultFetchRelays = listOf("wss://relay.nostr.band", "wss://relay.damus.io")
+    private val defaultMetadataRelays = listOf("wss://purplepag.es", "wss://user.kindpag.es")
+    private val defaultArticleFetchRelays = setOf("wss://nos.lol") + defaultFetchRelays
+
     /**
      * Parses all relevant information from a main site so duplicate calls aren't needed
      */
@@ -204,6 +236,142 @@ class FeedParser(override val di: DI) : DIAware {
      */
     private suspend fun curl(url: URL) = client.curl(url)
 
+    private suspend fun parseNostrUri(adaptedNostrUri: String): Nip19Profile {
+        val nostrUri = adaptedNostrUri.removePrefix("https://njump.me/")
+        if (nostrUri.contains("@")) { // It means it is a Nip05 address
+            val rawString = nostrUri.removePrefix("nostr:")
+            val parsedNip5 = getNip05Profile(rawString)
+            val (pubkey, relays) = parsedNip5.publicKey() to parsedNip5.relays()
+            return Nip19Profile(pubkey, relays)
+        } else {
+            val parsedProfile = Nip21.parse(nostrUri).asEnum()
+            when (parsedProfile) {
+                is Nip21Enum.Pubkey -> return Nip19Profile(parsedProfile.publicKey)
+                is Nip21Enum.Profile -> return Nip19Profile(parsedProfile.profile.publicKey(), parsedProfile.profile.relays())
+                else -> throw NostrUriParserException(message = "Could not find the user's info: $nostrUri")
+            }
+        }
+    }
+
+    suspend fun getProfileMetadata(nostrUri: URL): Either<MetaDataParseError, AuthorNostrData> {
+        return Either.catching(
+            onCatch = { t -> MetaDataParseError(url = nostrUri.toString(), throwable = t) },
+        ) {
+            val possibleNostrProfile = parseNostrUri(nostrUri.toString())
+            val publicKey = possibleNostrProfile.publicKey()
+            val relayList =
+                possibleNostrProfile.relays()
+                    .takeIf {
+                        it.size < 4
+                    }.orEmpty()
+                    .ifEmpty { getUserPublishRelays(publicKey) }
+            logDebug(LOG_TAG, "Relays from Nip19 -> ${relayList.joinToString(separator = ", ")}")
+            relayList
+                .ifEmpty { defaultFetchRelays }
+                .forEach { relayUrl ->
+                    nostrClient.addReadRelay(relayUrl)
+                }
+            nostrClient.connect()
+            val profileInfo =
+                try {
+                    nostrClient.fetchMetadata(
+                        publicKey = publicKey,
+                        timeout = Duration.ofSeconds(5L),
+                    )
+                } catch (e: NostrSdkException) {
+                    // We will use a default relay regardless of whether it is added above, to keep things simple.
+                    nostrClient.addReadRelay(defaultFetchRelays.random())
+                    nostrClient.connect()
+                    nostrClient.fetchMetadata(
+                        publicKey = publicKey,
+                        timeout = Duration.ofSeconds(5L),
+                    )
+                }
+            logDebug(LOG_TAG, profileInfo.asPrettyJson())
+
+            // Check if all relays in relaylist can be connected to
+            AuthorNostrData(
+                uri = possibleNostrProfile.toNostrUri(),
+                name = profileInfo.getName().toString(),
+                publicKey = publicKey,
+                imageUrl = profileInfo.getPicture().toString(),
+                relayList = nostrClient.relays().map { relayEntry -> relayEntry.key },
+            )
+        }
+    }
+
+    private suspend fun getUserPublishRelays(userPubkey: PublicKey): List<String> {
+        val userRelaysFilter =
+            Filter()
+                .author(userPubkey)
+                .kind(
+                    Kind.fromEnum(KindEnum.RelayList),
+                )
+
+        nostrClient.removeAllRelays()
+        defaultMetadataRelays.forEach { relayUrl ->
+            nostrClient.addReadRelay(relayUrl)
+        }
+        nostrClient.connect()
+        val potentialUserRelays =
+            nostrClient.fetchEventsFrom(
+                urls = defaultMetadataRelays,
+                filters = listOf(userRelaysFilter),
+                timeout = Duration.ofSeconds(5),
+            )
+        val relayList = extractRelayList(potentialUserRelays.toVec().first())
+        val relaysToUse =
+            if (relayList.any { (_, relayType) -> relayType == RelayMetadata.WRITE }) {
+                relayList.filter { it.value == RelayMetadata.WRITE }.map { entry -> entry.key }
+            } else if (relayList.size < 7) {
+                relayList.map { entry -> entry.key } // This represents the relay URL, just as the operation above.
+            } else {
+                defaultArticleFetchRelays.map { it }
+            }
+
+        return relaysToUse
+    }
+
+    private suspend fun fetchArticlesForAuthor(
+        author: PublicKey,
+        relays: List<String>,
+    ): List<Event> {
+        val articlesByAuthorFilter =
+            Filter()
+                .author(author)
+                .kind(Kind.fromEnum(KindEnum.LongFormTextNote))
+        logDebug(LOG_TAG, "Relay List size: ${relays.size}")
+
+        nostrClient.removeAllRelays()
+        val relaysToUse =
+            relays.take(3).plus(defaultArticleFetchRelays.random())
+                .ifEmpty { defaultFetchRelays }
+        relaysToUse.forEach { relay -> nostrClient.addReadRelay(relay) }
+        nostrClient.connect()
+        logDebug(LOG_TAG, "FETCHING ARTICLES")
+        val articleEventSet =
+            nostrClient.fetchEventsFrom(
+                urls = relaysToUse,
+                filters =
+                    listOf(
+                        articlesByAuthorFilter,
+                    ),
+                timeout = Duration.ofSeconds(10L),
+            ).toVec()
+        val articleEvents = articleEventSet.distinctBy { it.tags().find(TagKind.Title) }
+        nostrClient.removeAllRelays() // This is necessary to avoid piling relays to fetch from(on each fetch).
+        return articleEvents
+    }
+
+    suspend fun findNostrFeed(authorNostrData: AuthorNostrData): Either<FeedParserError, ParsedFeed> {
+        return Either.catching(
+            onCatch = { t -> FetchError(url = authorNostrData.uri, throwable = t) },
+        ) {
+            val fetchedArticles = fetchArticlesForAuthor(authorNostrData.publicKey, authorNostrData.relayList)
+            fetchedArticles.mapToFeed(authorNostrData.uri, authorNostrData.name, authorNostrData.imageUrl)
+        }
+    }
+
     suspend fun parseFeedUrl(url: URL): Either<FeedParserError, ParsedFeed> {
         return client.curlAndOnResponse(url) {
             parseFeedResponse(it)
@@ -286,6 +454,93 @@ class FeedParser(override val di: DI) : DIAware {
     }
 }
 
+private fun List<Event>.mapToFeed(
+    nostrUri: String,
+    authorName: String,
+    imageUrl: String,
+): ParsedFeed {
+    val description = "Nostr: $authorName"
+    val author =
+        ParsedAuthor(
+            name = authorName,
+            url = "https://njump.me/${first().author().toBech32()}",
+            avatar = imageUrl,
+        )
+    val articles =
+        this.sortedByDescending {
+            it.tags().find(TagKind.PublishedAt)?.content()?.toLong() ?: 0L
+        }.map { event: Event ->
+            event.asArticle(authorName, imageUrl)
+        }
+
+    return ParsedFeed(
+        title = authorName,
+        home_page_url = null,
+        feed_url = nostrUri,
+        description = description,
+        user_comment = null,
+        next_url = null,
+        icon = imageUrl,
+        favicon = null,
+        author = author,
+        expired = null,
+        items = articles,
+    )
+}
+
+fun Event.asArticle(
+    authorName: String,
+    imageUrl: String,
+): ParsedArticle {
+    val articleId = id().toString()
+    val articleUri = id().toNostrUri()
+    val articleNostrAddress =
+        Coordinate(
+            Kind.fromEnum(KindEnum.LongFormTextNote),
+            author(),
+            tags().find(
+                TagKind.SingleLetter(
+                    SingleLetterTag.lowercase(Alphabet.D),
+                ),
+            )?.content().toString(),
+        ).toBech32()
+    // Highlighter is a service for reading Nostr articles on the web.
+    val externalLink = "https://highlighter.com/a/$articleNostrAddress"
+    val articleAuthor =
+        ParsedAuthor(
+            name = authorName,
+            url = "https://njump.me/${author().toBech32()}",
+            avatar = imageUrl,
+        )
+    val articleTitle = tags().find(TagKind.Title)?.content()
+    val publishDate =
+        Instant.ofEpochSecond(
+            tags().find(TagKind.PublishedAt)?.content()?.toLong() ?: Instant.EPOCH.epochSecond,
+        ).atZone(ZoneId.systemDefault()).withFixedOffsetZone()
+    val articleTags = tags().hashtags()
+    val articleImage = tags().find(TagKind.Image)?.content()
+    val articleSummary = tags().find(TagKind.Summary)?.content()
+    val articleContent = content()
+    val parsedMarkdown = markDownParser.buildMarkdownTreeFromString(articleContent)
+    val htmlContent = HtmlGenerator(articleContent, parsedMarkdown, CommonMarkFlavourDescriptor()).generateHtml()
+
+    return ParsedArticle(
+        id = articleId,
+        url = articleUri,
+        external_url = externalLink,
+        title = articleTitle,
+        content_html = htmlContent,
+        content_text = articleContent,
+        summary = articleSummary,
+        image = if (articleImage != null) MediaImage(url = articleImage.toString()) else null,
+        date_published = publishDate.toString(),
+        date_modified = publishDate.toString(),
+        author = articleAuthor,
+        tags = articleTags,
+        attachments = null,
+    )
+}
+
 private fun GoFeed.asFeed(url: URL): ParsedFeed =
     ParsedFeed(
         title = title,
@@ -479,6 +734,22 @@ suspend fun <T> OkHttpClient.curlAndOnResponse(
     }
 }
 
+private val markDownParser = MarkdownParser(CommonMarkFlavourDescriptor())
+
+class AuthorNostrData(
+    val uri: String,
+    val name: String,
+    val publicKey: PublicKey,
+    val imageUrl: String,
+    val relayList: List<String>,
+)
+
+sealed class NostrException(message: String) : Exception(message)
+
+class NostrUriParserException(override val message: String) : NostrException(message)
+
+class NostrMetadataException(override val message: String) : NostrException(message)
+
 @Parcelize
 sealed class FeedParserError : Parcelable {
     abstract val url: String