crier

RSS and Atom feed aggregator
Info | Log | Files | Refs

commit 2f7df138569d2dfab7afa1ba5b2da83ae621aee1
parent 59af7e0d30f2744583a55f038b2a680a48020274
Author: lash <dev@holbrook.no>
Date:   Thu, 25 Jul 2024 21:20:11 +0100

Replace feed_rs crate with feed crate

Split repo in binary and lib, implement binary until metadata

Add public metadata setters

Change rss parser, include dublincore date in entry rss parse

Fully replaced feed_rs

Remove feed_rs from cargo

Remove commented code

Diffstat:
MCargo.lock | 187++++++++++++++++++++++++++++++++++++++++---------------------------------------
MCargo.toml | 42+++++++++++++-----------------------------
Acrier-lib/Cargo.lock | 829+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Acrier-lib/Cargo.toml | 45+++++++++++++++++++++++++++++++++++++++++++++
Rsrc/cache.rs -> crier-lib/src/cache.rs | 0
Acrier-lib/src/io.rs | 18++++++++++++++++++
Acrier-lib/src/io/fs.rs | 76++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Acrier-lib/src/lib.rs | 279+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Acrier-lib/src/log.rs | 21+++++++++++++++++++++
Rsrc/mem.rs -> crier-lib/src/mem.rs | 0
Acrier-lib/src/meta.rs | 67+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Acrier-lib/src/rss.rs | 197+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Acrier-lib/src/tests.rs | 263+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Rtestdata/test.atom.xml -> crier-lib/testdata/test.atom.xml | 0
Acrier-lib/testdata/test.rss.xml | 66++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Rtestdata/test2.xml -> crier-lib/testdata/test2.xml | 0
Dsrc/io.rs | 18------------------
Dsrc/io/fs.rs | 72------------------------------------------------------------------------
Dsrc/lib.rs | 356-------------------------------------------------------------------------------
Asrc/main.rs | 103+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Dsrc/meta.rs | 48------------------------------------------------
Dsrc/tests.rs | 251-------------------------------------------------------------------------------
22 files changed, 2071 insertions(+), 867 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock @@ -38,6 +38,7 @@ dependencies = [ [[package]] name = "atom_syndication" version = "0.12.3" +source = "git+git://holbrook.no/contrib/atom_syndication?rev=9985c1610b2b819f5bd2f7a719567ee0b5419b85#9985c1610b2b819f5bd2f7a719567ee0b5419b85" dependencies = [ "chrono", "derive_builder", @@ -70,12 +71,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] -name = "bitflags" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" - -[[package]] name = "block-buffer" version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -92,15 +87,15 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytes" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +checksum = "a12916984aab3fa6e39d655a33e09c0071eb36d6ab3aea5c2d78551f1df6d952" [[package]] name = "cc" -version = "1.0.98" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" +checksum = "2aba8f4e9906c7ce3c73463f62a7f0c65183ada1a2d47e397cc8810827f9694f" [[package]] name = "cfg-if" @@ -130,7 +125,7 @@ checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" dependencies = [ "ansi_term", "atty", - "bitflags 1.3.2", + "bitflags", "strsim 0.8.0", "textwrap", "unicode-width", @@ -149,21 +144,27 @@ version = "0.0.1" dependencies = [ "atom_syndication", "chrono", - "clap", "digest", "feed-rs", "http", "itertools", - "mediatype", - "quick-xml 0.31.0", "rs_sha512", "rss", "serde", - "tempfile", "uuid", ] [[package]] +name = "crier-bin" +version = "0.0.1" +dependencies = [ + "clap", + "crier", + "env_logger", + "log", +] + +[[package]] name = "crypto-common" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -175,9 +176,9 @@ dependencies = [ [[package]] name = "darling" -version = "0.20.9" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1" +checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" dependencies = [ "darling_core", "darling_macro", @@ -185,9 +186,9 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.9" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120" +checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" dependencies = [ "fnv", "ident_case", @@ -199,9 +200,9 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.20.9" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", @@ -274,30 +275,27 @@ dependencies = [ ] [[package]] -name = "errno" -version = "0.3.9" +name = "env_logger" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" dependencies = [ - "libc", - "windows-sys", + "atty", + "humantime", + "log", + "regex", + "termcolor", ] [[package]] -name = "fastrand" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" - -[[package]] name = "feed-rs" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be5366c3d4ae865540354ecafa0e5b41dd56c2f31d0b2ef876669edf964daaec" +checksum = "2c546f6f85e02eaca09e6a6dd22fe4db839745f55ae16c8f36626980a57f5bc4" dependencies = [ "chrono", "mediatype", - "quick-xml 0.31.0", + "quick-xml 0.36.1", "regex", "serde", "serde_json", @@ -363,6 +361,12 @@ dependencies = [ ] [[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] name = "iana-time-zone" version = "0.1.60" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -432,16 +436,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" [[package]] -name = "linux-raw-sys" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" - -[[package]] name = "log" -version = "0.4.21" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "mediatype" @@ -512,6 +510,16 @@ dependencies = [ ] [[package]] +name = "quick-xml" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96a05e2e8efddfa51a84ca47cec303fac86c8541b686d37cac5efc0e094417bc" +dependencies = [ + "encoding_rs", + "memchr", +] + +[[package]] name = "quote" version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -610,19 +618,6 @@ dependencies = [ ] [[package]] -name = "rustix" -version = "0.38.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" -dependencies = [ - "bitflags 2.6.0", - "errno", - "libc", - "linux-raw-sys", - "windows-sys", -] - -[[package]] name = "ryu" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -630,18 +625,18 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "serde" -version = "1.0.203" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" +checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.203" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" +checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2", "quote", @@ -650,9 +645,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.118" +version = "1.0.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d947f6b3163d8857ea16c4fa0dd4840d52f3041039a85decd46867eb1abef2e4" +checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" dependencies = [ "itoa", "ryu", @@ -679,9 +674,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "syn" -version = "2.0.68" +version = "2.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9" +checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af" dependencies = [ "proc-macro2", "quote", @@ -689,15 +684,12 @@ dependencies = [ ] [[package]] -name = "tempfile" -version = "3.10.1" +name = "termcolor" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" dependencies = [ - "cfg-if", - "fastrand", - "rustix", - "windows-sys", + "winapi-util", ] [[package]] @@ -711,9 +703,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.6.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c55115c6fbe2d2bef26eb09ad74bde02d8255476fc0c7b515ef09fbb35742d82" +checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" dependencies = [ "tinyvec_macros", ] @@ -770,9 +762,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.9.1" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de17fd2f7da591098415cff336e12965a28061ddace43b59cb3c430179c9439" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" dependencies = [ "getrandom", ] @@ -866,6 +858,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] +name = "winapi-util" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" +dependencies = [ + "windows-sys", +] + +[[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -891,9 +892,9 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", @@ -907,48 +908,48 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" diff --git a/Cargo.toml b/Cargo.toml @@ -1,41 +1,25 @@ [package] -name = "crier" +name = "crier-bin" version = "0.0.1" edition = "2021" rust-version = "1.79" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +authors = ["Louis Holbrook <dev@holbrook.no>"] [dependencies] -#rss = "^2.0.3" -feed-rs = "^2.0" -rss = "^2.0" -#activitypub_federation = "^0.4.0" -#json-ld = "^0.14.1" -digest = "^0.10.7" clap = "2.34.0" -#sha2 = "^0.10.8" -rs_sha512 = "^0.1.3" -http = "^1.0" -chrono = "^0.4" -uuid = "^1.9" -itertools = "^0.13" -serde = "^1.0" -atom_syndication = "^0.12" +crier = { path = "./crier-lib", features = ["fs"] } -#[dependencies.quick-xml] -#version = "^0.31" -#features = ["serialize"] +[dependencies.env_logger] +version = "^0.9" +optional = true -[patch.crates-io] -atom_syndication = { path = "/home/lash/src/contrib/atom_syndication" } -#atom_syndication = { git = "git://holbrook.no/contrib/atom_syndication", rev="fab6f733f6481ecfcbd6a76074f1038c79c854ae" } #branch="lash/entry-to-xml" -#atom_syndication = { git = "git://holbrook.no/contrib/atom_syndication", rev="9985c1610b2b819f5bd2f7a719567ee0b5419b85" } #branch="lash/entry-fromstr" +[dependencies.log] +version = "^0.4" +optional = true -[dev-dependencies] -tempfile = "3.3.0" -mediatype = "^0.19" -quick-xml = "^0.31" +[patch.crates-io] +#atom_syndication = { path = "/home/lash/src/contrib/atom_syndication" } +atom_syndication = { git = "git://holbrook.no/contrib/atom_syndication", rev="9985c1610b2b819f5bd2f7a719567ee0b5419b85" } #branch="lash/entry-fromstr" [features] -fs = [] +logging = ["log", "env_logger"] diff --git a/crier-lib/Cargo.lock b/crier-lib/Cargo.lock @@ -0,0 +1,829 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "atom_syndication" +version = "0.12.3" +source = "git+git://holbrook.no/contrib/atom_syndication?rev=9985c1610b2b819f5bd2f7a719567ee0b5419b85#9985c1610b2b819f5bd2f7a719567ee0b5419b85" +dependencies = [ + "chrono", + "derive_builder", + "diligent-date-parser", + "never", + "quick-xml 0.32.0", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" + +[[package]] +name = "bitflags" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + +[[package]] +name = "bytes" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" + +[[package]] +name = "cc" +version = "1.0.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-targets", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" + +[[package]] +name = "crier" +version = "0.0.1" +dependencies = [ + "atom_syndication", + "chrono", + "digest", + "env_logger", + "http", + "itertools", + "log", + "mediatype", + "quick-xml 0.31.0", + "rs_sha512", + "rss", + "serde", + "tempfile", + "uuid", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "darling" +version = "0.20.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.20.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "derive_builder" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0350b5cb0331628a5916d6c5c0b72e97393b8b6b03b47a9284f4e7f5a405ffd7" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d48cda787f839151732d396ac69e3473923d54312c070ee21e9effcaa8ca0b1d" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b" +dependencies = [ + "derive_builder_core", + "syn", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "diligent-date-parser" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6cf7fe294274a222363f84bcb63cdea762979a0443b4cf1f4f8fd17c86b1182" +dependencies = [ + "chrono", +] + +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + +[[package]] +name = "encoding_rs" +version = "0.8.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "env_logger" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "errno" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "iana-time-zone" +version = "0.1.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "js-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.155" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" + +[[package]] +name = "linux-raw-sys" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + +[[package]] +name = "log" +version = "0.4.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" + +[[package]] +name = "mediatype" +version = "0.19.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8878cd8d1b3c8c8ae4b2ba0a36652b7cf192f618a599a7fbdfa25cffd4ea72dd" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "never" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96aba5aa877601bb3f6dd6a63a969e1f82e60646e81e71b14496995e9853c91" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "proc-macro2" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quick-xml" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" +dependencies = [ + "encoding_rs", + "memchr", +] + +[[package]] +name = "quick-xml" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d3a6e5838b60e0e8fa7a43f22ade549a37d61f8bdbe636d0d7816191de969c2" +dependencies = [ + "encoding_rs", + "memchr", +] + +[[package]] +name = "quote" +version = "1.0.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" + +[[package]] +name = "rs_hasher_ctx" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a45ae5cc6246fa2666253289d6495e1fb3d125fb83842ff56b747a3b662e28e" +dependencies = [ + "rs_internal_hasher", + "rs_internal_state", + "rs_n_bit_words", +] + +[[package]] +name = "rs_internal_hasher" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19754b7c13d7fb92e995b1f6330918466e134ba7c3f55bf805c72e6a9727c426" +dependencies = [ + "rs_internal_state", + "rs_n_bit_words", +] + +[[package]] +name = "rs_internal_state" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "214a4e27fec5b651d615675874c6a829496cc2aa66e5f1b184ab05cb39fd3625" +dependencies = [ + "rs_n_bit_words", +] + +[[package]] +name = "rs_n_bit_words" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bc1bbb4c2a60f76b331e6ba70b5065e210fa6e72fc966c2d488736755d89cb6" + +[[package]] +name = "rs_sha512" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78bb3ee2bcf2e0bd2ead2504c3b67d1fb34ae978a2014febc011f82fcbe58d56" +dependencies = [ + "rs_hasher_ctx", + "rs_internal_hasher", + "rs_internal_state", + "rs_n_bit_words", +] + +[[package]] +name = "rss" +version = "2.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f374fd66bb795938b78c021db1662d43a8ffbc42ec1ac25429fc4833b732751" +dependencies = [ + "atom_syndication", + "derive_builder", + "never", + "quick-xml 0.31.0", +] + +[[package]] +name = "rustix" +version = "0.38.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "serde" +version = "1.0.203" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.203" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +dependencies = [ + "cfg-if", + "fastrand", + "rustix", + "windows-sys", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "uuid" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5de17fd2f7da591098415cff336e12965a28061ddace43b59cb3c430179c9439" +dependencies = [ + "getrandom", +] + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" diff --git a/crier-lib/Cargo.toml b/crier-lib/Cargo.toml @@ -0,0 +1,45 @@ +[package] +name = "crier" +version = "0.0.1" +edition = "2021" +rust-version = "1.79" +authors = ["Louis Holbrook <dev@holbrook.no>"] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +digest = "^0.10.7" +rs_sha512 = "^0.1.3" +http = "^1.0" +chrono = "^0.4" +itertools = "^0.13" +serde = "^1.0" +atom_syndication = "^0.12" + +[dependencies.uuid] +version = "^1.9" +features = ["v4"] + +[dependencies.rss] +version = "^2.0" +features = ["atom"] + +[patch.crates-io] +atom_syndication = { git = "git://holbrook.no/contrib/atom_syndication", rev="9985c1610b2b819f5bd2f7a719567ee0b5419b85" } #branch="lash/entry-fromstr" + +[dev-dependencies] +tempfile = "3.3.0" +mediatype = "^0.19" +quick-xml = "^0.31" + +[dependencies.env_logger] +version = "^0.9" +optional = true + +[dependencies.log] +version = "^0.4" +optional = true + +[features] +fs = [] +logging = ["log", "env_logger"] diff --git a/src/cache.rs b/crier-lib/src/cache.rs diff --git a/crier-lib/src/io.rs b/crier-lib/src/io.rs @@ -0,0 +1,18 @@ +use atom_syndication::Feed; + +pub enum FeedMethod { + Read, + Create, + Update, +} + +pub trait FeedGet { + fn get(&self, s: &str, method: Option<FeedMethod>) -> Result<Feed, u64>; +} + +pub trait FeedPut { + fn put(&self, feed: &Feed, s: &str, method: Option<FeedMethod>) -> u64; +} + +#[cfg(feature = "fs")] +pub mod fs; diff --git a/crier-lib/src/io/fs.rs b/crier-lib/src/io/fs.rs @@ -0,0 +1,76 @@ +use std::path::Path; +use std::path::PathBuf; +use std::fs::File; +use std::collections::HashMap; +use std::io::Write; + +use atom_syndication::Feed; + +use super::FeedMethod; +use super::FeedGet; +use crate::cache::Cache; +use crate::rss::from_file as rss_from_file; + + +pub struct FsFeed { +} + +pub struct FsCache { + dir: PathBuf, + files: HashMap<String, File>, +} + +impl FeedGet for FsFeed { + fn get(&self, s: &str, _method: Option<FeedMethod>) -> Result<Feed, u64> { + let feed: Feed; + match rss_from_file(s, false) { + Ok(v) => { + feed = v; + }, + Err(e) => { + return Err(0); + }, + }; + Ok(feed) + } +} + +impl FsCache { + pub fn new(path: PathBuf) -> FsCache { + FsCache{ + dir: path, + files: HashMap::new(), + } + } +} + +impl Cache for FsCache { + fn open(&mut self, id: String) -> &mut dyn Write { + let p: &Path; + let fp: PathBuf; + let mut s: String; + let mut ids: String; + let mut f: File; + + if !self.files.contains_key(&id) { + p = Path::new(self.dir.as_path()); + + ids = id.clone(); + ids = ids.replace("/", "%2F"); + ids = ids.replace("\\", "%5C"); + + fp = p.join(ids); + s = String::from(fp.to_str().unwrap()); + f = File::create(s.as_str()).unwrap(); + self.files.insert(id.clone(), f); + } + return self.files.get_mut(&id).unwrap(); + } + + fn close(&mut self, id: String) -> usize { + if self.files.contains_key(&id) { + self.files.remove(&id); + } + 0 + } +} diff --git a/crier-lib/src/lib.rs b/crier-lib/src/lib.rs @@ -0,0 +1,279 @@ +use std::collections::HashMap; +use std::hash::Hasher; +use std::hash::Hash; +use std::iter::Iterator; +use std::io::Write; +use std::fmt::Debug; +use std::io::BufWriter; +use std::str::FromStr; + +use rs_sha512::Sha512Hasher; +use chrono::Local; +use atom_syndication::Feed as Feed; +use atom_syndication::Entry as Entry; +use atom_syndication::TextType as OutTextType; +use atom_syndication::Text as OutText; +use atom_syndication::Content as OutContent; +use atom_syndication::Person as OutPerson; +use atom_syndication::Category as OutCategory; +use atom_syndication::FixedDateTime; +use atom_syndication::Person; +use itertools::Itertools; + +pub mod io; +pub mod mem; + +mod meta; +mod cache; +mod rss; +mod log; +use meta::FeedMetadata; +use mem::CacheWriter; +use cache::Cache; + +#[derive(Debug)] +pub enum Error { + WriteError, + CacheError, + ParseError, + IncompleteError, +} + +pub struct Sequencer<'a> { + metadata: FeedMetadata, + pub items: HashMap<u64, Vec<u8>>, + item_keys: Vec<u64>, + crsr: usize, + limit: usize, + default_cache: CacheWriter, //HashMap<String, Vec<u8>>, + cache: Option<&'a mut dyn Cache>, +} + +pub struct SequencerEntry { + pub digest: u64, + entry: Entry, + out: Vec<u8>, +} + +impl<'a> Sequencer<'a> { + pub fn new() -> Sequencer<'a> { + let mut o = Sequencer { + metadata: FeedMetadata::default(), + items: HashMap::new(), + crsr: 0, + limit: 0, + item_keys: Vec::new(), + default_cache: CacheWriter::new(), //HashMap::new(), + cache: None, + }; + + #[cfg(test)] + o.metadata.force(); + + o + } + + pub fn with_cache(mut self, w: &'a mut impl Cache) -> Sequencer<'a> { + self.cache = Some(w); + return self; + } + + pub fn set_author(&mut self, name: &str) -> bool { + self.metadata.set_author(Person{ + name: String::from(name), + email: None, + uri: None, + }) + } + + pub fn set_title(&mut self, title: &str) -> bool { + self.metadata.set_title(String::from(title)) + } + + pub fn add(&mut self, entry: Entry) -> bool { + let w: &mut dyn Write; + let mut id: String; + + id = entry.id.to_string(); + match &mut self.cache { + Some(v) => { + w = v.open(id); + }, + None => { + w = &mut self.default_cache; + }, + } + + id = entry.id.to_string(); + let o = SequencerEntry::new(entry, w); + if self.items.contains_key(&o.digest) { + return false; + } + self.items.insert(o.digest, o.into()); + match &mut self.cache { + Some(v) => { + v.close(id); + }, + None => { + }, + } + return true; + } + + pub fn add_from(&mut self, feed: Feed) -> i64 { + let mut c: i64; + + c = 0; + for v in feed.entries.iter() { + self.add(v.clone()); + c += 1; + } + c + } + + pub fn write_to(&mut self, w: impl Write) -> Result<usize, Error> { + let mut r: usize; + let mut feed = Feed::default(); + let mut entry: Entry; + let mut entries: Vec<Entry>; + let mut b: &str; + feed.set_id("urn:uuid:60a76c80-d399-11d9-b91C-0003939e0af6"); + feed.set_updated(Local::now().to_utc()); + + match self.metadata.apply(&mut feed) { + Err(_v) => { + return Err(Error::WriteError); + }, + Ok(_) => { + }, + } + + entries = Vec::new(); + r = 0; + for v in self { + b = std::str::from_utf8(v.as_slice()).unwrap(); + match Entry::from_str(b) { + Err(e) => { + println!("fromstrerr {:?}", e); + return Err(Error::CacheError); + }, + Ok(o) => { + entries.push(o); + }, + } + r += 1; + } + feed.set_entries(entries); + + match feed.write_to(w) { + Err(_v) => { + return Err(Error::WriteError); + }, + Ok(_) => { + }, + } + + Ok(r) + } +} + +impl<'a> Iterator for Sequencer<'a> { + type Item = Vec<u8>; + + fn next(&mut self) -> Option<Self::Item> { + let c: u64; + + if self.limit == 0 { + self.item_keys = Vec::new(); + for k in self.items.keys().sorted() { + self.item_keys.push(k.clone()); + self.limit += 1; + } + } + + if self.limit == 0 { + return None; + } + + if self.crsr == self.limit { + self.limit = 0; + self.crsr = 0; + return None; + } + + c = self.item_keys[self.crsr]; + self.crsr += 1; + return Some(self.items[&c].clone()); + } +} + +impl SequencerEntry { + pub fn new(entry: Entry, exporter: &mut dyn Write) -> SequencerEntry { + let mut have_date: bool; + let mut id_part: u32; + let mut o = SequencerEntry { + entry: entry, + digest: 0, + out: Vec::new(), + }; + + have_date = false; + match &o.entry.published { + Some(v) => { + id_part = v.timestamp() as u32; + o.digest = id_part as u64; + o.digest <<= 32; + have_date = true; + }, + None => { + }, + } + + if !have_date { + id_part = o.entry.updated.timestamp() as u32; + o.digest = id_part as u64; + o.digest <<= 32; + have_date = true; + } + + let mut h = Sha512Hasher::default(); + o.hash(&mut h); + id_part = h.finish() as u32; + o.digest += id_part as u64; + o + } + + /// TODO: get size heuristics from already written values (either that or replace underlying + /// in-memory writer implementation with something that doesnt wrap on flush. + fn to_writer(&self, v: Vec<u8>) -> BufWriter<Vec<u8>> { + BufWriter::with_capacity(10241024, v) + } + +} + +/// TODO: split out field translations to separate module +impl Into<Vec<u8>> for SequencerEntry { + fn into(self) -> Vec<u8> { + let mut out_entry: Entry; + let mut b: Vec<u8>; + let mut w: BufWriter<Vec<u8>>; + let o: &SequencerEntry; + + o = &self; + b = Vec::new(); + w = o.to_writer(b); + + w = self.entry.write_to(w).unwrap(); + b = Vec::from(w.buffer()); + b + } +} + +impl Hash for SequencerEntry { + fn hash<H: Hasher>(&self, h: &mut H) { + h.write(self.entry.id.as_bytes()); + } +} + +#[cfg(test)] +mod tests; diff --git a/crier-lib/src/log.rs b/crier-lib/src/log.rs @@ -0,0 +1,21 @@ +#[cfg(feature = "logging")] +use env_logger; +pub use log::debug; +pub use log::info; + +pub fn init() { + env_logger::init(); +} + +#[cfg(not(feature = "logging"))] +#[macro_export] +macro_rules! info { + (*) => {}; +} + +#[cfg(not(feature = "logging"))] +#[macro_export] +macro_rules! debug { + (*) => {}; +} + diff --git a/src/mem.rs b/crier-lib/src/mem.rs diff --git a/crier-lib/src/meta.rs b/crier-lib/src/meta.rs @@ -0,0 +1,67 @@ +use atom_syndication::Person; +use atom_syndication::Feed; +use uuid::Uuid; + +#[derive(Debug)] +pub enum Error { + IncompleteFeedMetadata, +} + + +pub struct FeedMetadata { + pub author: Person, + pub title: String, + pub id: String, + flag: u8, +} + +impl Default for FeedMetadata { + fn default() -> FeedMetadata { + FeedMetadata{ + author: Person{ + name: "?".to_string(), + email: Some("?".to_string()), + uri: Some("?".to_string()), + }, + title: String::from("?"), + id: Uuid::new_v4().to_string(), + flag: 0, + } + } + + +} + +impl FeedMetadata { + pub fn force(&mut self) { + self.flag |= 3; + } + + fn check_complete(&self) -> bool { + self.flag >= 3 + } + + pub fn set_author(&mut self, author: Person) -> bool { + self.author = author; + self.flag |= 1; + self.check_complete() + } + + + pub fn set_title(&mut self, title: String) -> bool { + self.title = title; + self.flag |= 2; + self.check_complete() + } + + pub fn apply(&self, feed: &mut Feed) -> Result<(), Error> { + if !self.check_complete() { + return Err(Error::IncompleteFeedMetadata); + } + let mut persons = Vec::<Person>::new(); + persons.push(self.author.clone()); + feed.set_authors(persons); + feed.set_title(self.title.clone()); + Ok(()) + } +} diff --git a/crier-lib/src/rss.rs b/crier-lib/src/rss.rs @@ -0,0 +1,197 @@ +use std::path::Path; +use std::fs::File; +use std::io::BufReader; +use std::io::BufRead; +use crate::Error; +use crate::log::info; +use crate::log::debug; + +use rss::Channel; +use rss::Item; +use rss::extension::dublincore::DublinCoreExtension; +use atom_syndication::Feed; +use atom_syndication::Entry; +use atom_syndication::Text; +use atom_syndication::TextType; +use atom_syndication::FixedDateTime; +use chrono::naive::NaiveDateTime; +use chrono::Local; +use chrono::offset::Utc; + +/// try to coerce the item field into a valid date +fn parse_date(v: &String) -> Result<FixedDateTime, Error> { + match FixedDateTime::parse_from_rfc2822(v.as_str()) { + Ok(r) => { + return Ok(r); + }, + Err(e) => {}, + }; + match FixedDateTime::parse_from_rfc3339(v.as_str()) { + Ok(r) => { + return Ok(r); + }, + Err(e) => {}, + }; + match FixedDateTime::parse_from_str(v.as_str(), "%Y-%m-%dT%H:%M:%S") { + Ok(r) => { + return Ok(r); + }, + Err(e) => { + }, + }; + match NaiveDateTime::parse_from_str(v.as_str(), "%Y-%m-%dT%H:%M:%S") { + Ok(r) => { + return Ok(r.and_utc().fixed_offset()); + }, + Err(e) => { + }, + }; + + + Err(Error::ParseError) +} + +/// try different item fields to determine the date +fn get_base_date(ipt: &Item) -> Result<FixedDateTime, Error> { + let mut ds = String::new(); + + match &ipt.pub_date { + Some(v) => { + ds.push_str(v.as_str()); + }, + _ => {}, + }; + match parse_date(&ds) { + Ok(v) => { + return Ok(v); + }, + Err(e) => {}, + }; + + match &ipt.dublin_core_ext { + Some(v) => { + for vv in v.dates() { + match parse_date(vv) { + Ok(vvv) => { + return Ok(vvv); + }, + Err(e) => { + debug!("no date"); + }, + } + } + }, + _ => {}, + } + + Err(Error::IncompleteError) +} + +/// coerce the rss item into an atom entry +fn translate_item(ipt: Item) -> Result<Entry, Error> { + let mut opt = Entry::default(); + + match &ipt.title { + Some(v) => { + opt.set_title(Text::plain(v)); + }, + _ => {}, + }; + + match get_base_date(&ipt) { + Ok(v) => { + opt.set_published(v); + }, + Err(e) => { + return Err(e); + } + } + Ok(opt) +} + + +fn translate(ipt: Channel, allow_fail: bool) -> Result<Feed, Error> { + let mut entries: Vec<Entry>; + let mut opt = Feed::default(); + + opt.set_title(Text::plain(&ipt.title)); + + opt.set_subtitle(Some(Text::plain(&ipt.description))); + + entries = vec!(); + for v in ipt.into_items() { + match translate_item(v) { + Ok(v) => { + entries.push(v); + }, + Err(e) => { + if !allow_fail { + return Err(Error::IncompleteError); + } + }, + } + } + + opt.set_entries(entries); + opt.set_updated(Local::now().to_utc()); + Ok(opt) +} + +pub fn from_file(fp: &str, allow_entry_fail: bool) -> Result<Feed, Error> { + let mut o: Channel; + let r: Feed; + let p: &Path; + let mut f: File; + //let mut b: BufReader; // how to explicitly declare + + p = Path::new(fp); + f = File::open(p).unwrap(); + let mut b = BufReader::new(f); + + match Feed::read_from(b) { + Ok(v) => { + return Ok(v); + }, + Err(e) => {}, + }; + + f = File::open(p).unwrap(); + b = BufReader::new(f); + + match Channel::read_from(b) { + Ok(v) => { + o = v; + }, + Err(e) => { + return Err(Error::ParseError); + }, + }; + o.set_dublin_core_ext(DublinCoreExtension::default()); + translate(o, allow_entry_fail) +} + +mod test { + use std::path::Path; + use atom_syndication::Feed; + use crate::log; + + #[test] + fn test_rss_from_file() { + env_logger::init(); + let mut r: Feed; + match super::from_file("testdata/test.rss.xml", false) { + Ok(v) => { + }, + Err(e) => { + panic!("{:?}", e); + }, + }; + match super::from_file("testdata/test.atom.xml", false) { + Ok(v) => { + }, + Err(e) => { + panic!("expected fail"); + }, + }; + } +} diff --git a/crier-lib/src/tests.rs b/crier-lib/src/tests.rs @@ -0,0 +1,263 @@ +use std::clone::Clone; +use std::fs::File; +use std::io::{SeekFrom, Seek, Read}; +use std::str; + +use mediatype::MediaTypeBuf; +use chrono::DateTime; +use tempfile::NamedTempFile; +use tempfile::tempdir; +use atom_syndication::Entry as OutEntry; +use atom_syndication::Feed as OutFeed; +use atom_syndication::Person; +use atom_syndication::Text; +use quick_xml::Reader as XMLReader; +use quick_xml::events::Event as XMLEvent; + +use crate::Sequencer; +use crate::io::FeedGet; +use crate::meta::FeedMetadata; +use crate::Feed; +use crate::Entry; +use crate::io::fs::FsCache; +use crate::mem::MemCache; + +#[cfg(feature = "fs")] +use crate::io::fs::FsFeed; + + +fn check_xml_title(xml: Vec<u8>, title: &str) { + let mut rxml = XMLReader::from_str(str::from_utf8(&xml).unwrap()); + let mut xmlbuf = Vec::new(); + let mut state = 0; + loop { + match rxml.read_event_into(&mut xmlbuf) { + Err(e) => panic!("cant read back xml: {:?}", e), + Ok(XMLEvent::Eof) => break, + Ok(XMLEvent::Start(v)) => { + match v.name().as_ref() { + b"title" => { + state = 1; + }, + _ => { + state = 0 + }, + } + }, + Ok(XMLEvent::End(v)) => { + state = 0; + }, + Ok(XMLEvent::Text(v)) => { + if state > 0 { + assert_eq!(v.unescape().unwrap(), title); + } + }, + _ => (), + } + } +} + +#[test] +fn test_entry_guard() { + let mut r: bool; + let mut seq = Sequencer::new(); + let mut src = Entry::default(); + let mut s: String; + + src.id = String::from("foo"); + s = String::from("inky"); + src.title = Text::plain(s); + + src.published = Some(DateTime::parse_from_rfc3339("2024-06-25T20:46:00+02:00").unwrap().into()); + r = seq.add(src); + assert!(r); + + let mut src_two = Entry::default(); + src_two.id = String::from("foo"); + s = String::from("pinky"); + src_two.title = Text::plain(s); + src_two.published = Some(DateTime::parse_from_rfc3339("2024-06-25T20:46:00+02:00").unwrap().into()); + r = seq.add(src_two); + assert!(!r); + + let mut src_three = Entry::default(); + src_three.id = String::from("foo"); + s = String::from("blinky"); + src_three.title = Text::plain(s); + + src_three.published = Some(DateTime::parse_from_rfc3339("2024-06-25T20:46:00+03:00").unwrap().into()); + r = seq.add(src_three); + assert!(r); +} + +#[test] +#[cfg(feature = "fs")] +fn test_feed_get() { + let r: bool; + let fs = FsFeed{}; + let feed = fs.get("testdata/test.atom.xml", None).unwrap(); + let mut seq = Sequencer::new(); + //r = seq.add(feed.entries.get(0).unwrap().clone()); + //assert!(r); +} + +#[test] +#[cfg(feature = "fs")] +fn test_feed_all() { + let r: i64; + let fs = FsFeed{}; + let feed = fs.get("testdata/test.atom.xml", None).unwrap(); + let mut seq = Sequencer::new(); + r = seq.add_from(feed); + assert_eq!(r, 16); +} + +#[test] +#[cfg(feature = "fs")] +fn test_feed_mix() { + let mut r: i64; + let fs = FsFeed{}; + let mut feed = fs.get("testdata/test.atom.xml", None).unwrap(); + let mut seq = Sequencer::new(); + r = seq.add_from(feed); + assert_eq!(r, 16); + feed = fs.get("testdata/test2.xml", None).unwrap(); + r = seq.add_from(feed); + assert_eq!(r, 10); + assert_eq!(seq.by_ref().count(), 26); + assert_eq!(seq.count(), 26); +} + +#[test] +#[cfg(feature = "fs")] +fn test_feed_write() { + let r: usize; + let fs = FsFeed{}; + let f: NamedTempFile; + let mut fr: File; + + let feed = fs.get("testdata/test.atom.xml", None).unwrap(); + let mut seq = Sequencer::new(); + seq.add_from(feed); + f = NamedTempFile::new().unwrap(); + fr = f.reopen().unwrap(); + r = seq.write_to(f).unwrap(); + assert_eq!(r, 16); + assert_eq!(fr.metadata().unwrap().len(), 520327); +} + +#[test] +#[cfg(feature = "fs")] +fn test_feed_write_extcache() { + let r: usize; + let fs = FsFeed{}; + let f: NamedTempFile; + let fr: File; + let mut cache: FsCache; + + let d = tempdir().unwrap(); + cache = FsCache::new(d.into_path()); + + let feed = fs.get("testdata/test.atom.xml", None).unwrap(); + let mut seq = Sequencer::new(); + seq = seq.with_cache(&mut cache); + + seq.add_from(feed); + f = NamedTempFile::new().unwrap(); + fr = f.reopen().unwrap(); + r = seq.write_to(f).unwrap(); + + assert_eq!(r, 16); + assert_eq!(fr.metadata().unwrap().len(), 520327); +} + +#[test] +#[cfg(feature = "fs")] +fn test_sequence_order() { + let mut seq = Sequencer::new(); + let mut entry: Entry; + let mut s: String; + let mut r: Vec<u8>; + + entry = Entry::default(); + entry.id = String::from("y"); + s = String::from("inky"); + entry.title = Text::plain(s); + entry.published = Some(DateTime::parse_from_rfc3339("2024-06-25T20:46:00+02:00").unwrap().into()); + seq.add(entry); + + + entry = Entry::default(); + entry.id = String::from("b"); + s = String::from("pinky"); + entry.title = Text::plain(s); + entry.published = Some(DateTime::parse_from_rfc3339("2023-06-25T20:46:00+02:00").unwrap().into()); + seq.add(entry); + + entry = Entry::default(); + entry.id = String::from("d"); + s = String::from("blinky"); + entry.title = Text::plain(s); + entry.published = Some(DateTime::parse_from_rfc3339("2024-06-25T20:46:00+02:00").unwrap().into()); + seq.add(entry); + + entry = Entry::default(); + entry.id = String::from("a"); + s = String::from("clyde"); + entry.title = Text::plain(s); + entry.published = Some(DateTime::parse_from_rfc3339("2024-06-25T20:46:00+02:00").unwrap().into()); + seq.add(entry); + + + // TODO find value where sort digest is reverse of lexical id + r = seq.next().unwrap(); + check_xml_title(r, "pinky"); + r = seq.next().unwrap(); + check_xml_title(r, "blinky"); + r = seq.next().unwrap(); + check_xml_title(r, "inky"); + r = seq.next().unwrap(); + check_xml_title(r, "clyde"); +} + +#[test] +fn test_meta() { + let mut o = FeedMetadata::default(); + let mut feed = OutFeed::default(); + + match o.apply(&mut feed) { + Ok(r) => { + panic!("metadata should not be ready"); + }, + Err(e) => {}, + }; + + o.set_title(String::from("foo")); + match o.apply(&mut feed) { + Ok(r) => { + panic!("metadata should not be ready"); + }, + Err(e) => {}, + }; + + o.set_author(Person{ + name: String::from("Foo Bar"), + email: Some("foo@bar.com".to_string()), + uri: Some("foo.bar.com".to_string()), + } + ); + o.apply(&mut feed).unwrap(); +} + +#[test] +fn test_rss() { +let fs = FsFeed{}; + let mut cache = MemCache::new(); + let fs = FsFeed{}; + + let feed = fs.get("testdata/test.rss.xml", None).unwrap(); + let mut seq = Sequencer::new(); + seq = seq.with_cache(&mut cache); + + seq.add_from(feed); +} diff --git a/testdata/test.atom.xml b/crier-lib/testdata/test.atom.xml diff --git a/crier-lib/testdata/test.rss.xml b/crier-lib/testdata/test.rss.xml @@ -0,0 +1,65 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<rdf:RDF + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns="http://purl.org/rss/1.0/" + xmlns:admin="http://webns.net/mvcb/" + xmlns:content="http://purl.org/rss/1.0/modules/content/" + xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:syn="http://purl.org/rss/1.0/modules/syndication/" + xmlns:taxo="http://purl.org/rss/1.0/modules/taxonomy/" +> + +<channel rdf:about="https://holbrook.no"> +<title>Bluto won&#x27;t get Olivia</title> +<link>https://holbrook.no</link> +<description>This is the default summary of the project</description> +<dc:date>2024-07-27T23:54:25</dc:date> +<dc:publisher>Louis Holbrook</dc:publisher> +<dc:creator>Louis Holbrook</dc:creator> +<items> + <rdf:Seq> + <rdf:li rdf:resource="https://holbrook.no/share/releases/bluto/bluto-0.0.2-alpha.1+build.7ed4da0449182876aad5e79f6ba8ed1e06766c12.tar.gz" /> + </rdf:Seq> +</items> +</channel> +<item rdf:about="https://holbrook.no/share/releases/bluto/bluto-0.0.2-alpha.1+build.7ed4da0449182876aad5e79f6ba8ed1e06766c12.tar.gz"> +<title>bluto 0.0.2-alpha.1</title> +<link>https://holbrook.no/share/releases/bluto/bluto-0.0.2-alpha.1+build.7ed4da0449182876aad5e79f6ba8ed1e06766c12.tar.gz</link> +<description>Release announcement: Bluto won&#x27;t get Olivia +============================================= +Version release: 0.0.2-alpha.1 + +License: perl + +Source bundles +-------------- +* https://holbrook.no/share/releases/bluto/bluto-0.0.2-alpha.1+build.7ed4da0449182876aad5e79f6ba8ed1e06766c12.tar.gz + + +VCS +--- +* git+git://git.defalsify.org/bluto.git + + +ONLINE RESOURCES +---------------- +* https://holbrook.no +* https://git.defalsify.org + + +CHANGELOG +--------- +* foo bar baz +* xyzzy foo foo +* inky pinky blinky sue + +----- + +Generated by bluto v0.0.1 (perl v5.38.2) at 2024-07-27T23:54:24Z + +----- +</description> +<dc:date>2024-07-27T23:54:24</dc:date> +</item> +</rdf:RDF> +\ No newline at end of file diff --git a/testdata/test2.xml b/crier-lib/testdata/test2.xml diff --git a/src/io.rs b/src/io.rs @@ -1,18 +0,0 @@ -use feed_rs::model::Feed; - -pub enum FeedMethod { - Read, - Create, - Update, -} - -pub trait FeedGet { - fn get(&self, s: &str, method: Option<FeedMethod>) -> Result<Feed, u64>; -} - -pub trait FeedPut { - fn put(&self, feed: &Feed, s: &str, method: Option<FeedMethod>) -> u64; -} - -#[cfg(feature = "fs")] -pub mod fs; diff --git a/src/io/fs.rs b/src/io/fs.rs @@ -1,72 +0,0 @@ -use super::FeedMethod; -use super::FeedGet; -use feed_rs::model::Feed; -use feed_rs::parser; -//use http::Uri; -use std::path::Path; -use std::path::PathBuf; -use std::fs::File; -use std::collections::HashMap; -//use core::str::FromStr; -//use std::io::stderr; -use std::io::Write; - -use crate::cache::Cache; - - -pub struct FsFeed { -} - -pub struct FsCache { - dir: PathBuf, - files: HashMap<String, File>, -} - -impl FeedGet for FsFeed { - fn get(&self, s: &str, _method: Option<FeedMethod>) -> Result<Feed, u64> { - //let uri = Uri::from_str(s).unwrap(); - let f = File::open(s).unwrap(); - let feed = parser::parse(f).unwrap(); - Ok(feed) - } -} - -impl FsCache { - pub fn new(path: PathBuf) -> FsCache { - FsCache{ - dir: path, - files: HashMap::new(), - } - } -} - -impl Cache for FsCache { - fn open(&mut self, id: String) -> &mut dyn Write { - let p: &Path; - let fp: PathBuf; - let mut s: String; - let mut ids: String; - let mut f: File; - - if !self.files.contains_key(&id) { - p = Path::new(self.dir.as_path()); - - ids = id.clone(); - ids = ids.replace("/", "%2F"); - ids = ids.replace("\\", "%5C"); - - fp = p.join(ids); - s = String::from(fp.to_str().unwrap()); - f = File::create(s.as_str()).unwrap(); - self.files.insert(id.clone(), f); - } - return self.files.get_mut(&id).unwrap(); - } - - fn close(&mut self, id: String) -> usize { - if self.files.contains_key(&id) { - self.files.remove(&id); - } - 0 - } -} diff --git a/src/lib.rs b/src/lib.rs @@ -1,356 +0,0 @@ -use std::collections::HashMap; -use std::hash::Hasher; -use std::hash::Hash; -use std::iter::Iterator; -use std::io::Write; -use std::fmt::Debug; -use std::io::BufWriter; -use std::str::FromStr; - -use feed_rs::model::Entry; -use feed_rs::model::Feed; -use rs_sha512::Sha512Hasher; -//use chrono::DateTime; -use chrono::Local; -use atom_syndication::Feed as OutFeed; -use atom_syndication::Entry as OutEntry; -use atom_syndication::TextType as OutTextType; -use atom_syndication::Text as OutText; -use atom_syndication::Content as OutContent; -use atom_syndication::Person as OutPerson; -use atom_syndication::Category as OutCategory; -use atom_syndication::FixedDateTime; -use itertools::Itertools; - -mod meta; -mod io; -mod mem; -mod cache; -use meta::FeedMetadata; -use mem::CacheWriter; -use cache::Cache; - -#[derive(Debug)] -pub enum Error { - WriteError, - CacheError, -} - - -pub struct Sequencer<'a> { - metadata: FeedMetadata, - pub items: HashMap<u64, Vec<u8>>, - item_keys: Vec<u64>, - crsr: usize, - limit: usize, - default_cache: CacheWriter, //HashMap<String, Vec<u8>>, - cache: Option<&'a mut dyn Cache>, -} - -pub struct SequencerEntry { - pub digest: u64, - entry: Entry, - out: Vec<u8>, -} - -impl<'a> Sequencer<'a> { - pub fn new() -> Sequencer<'a> { - let mut o = Sequencer { - metadata: FeedMetadata::default(), - items: HashMap::new(), - crsr: 0, - limit: 0, - item_keys: Vec::new(), - default_cache: CacheWriter::new(), //HashMap::new(), - cache: None, - }; - - #[cfg(test)] - o.metadata.force(); - - o - } - - pub fn with_cache(mut self, w: &'a mut impl Cache) -> Sequencer<'a> { - self.cache = Some(w); - return self; - } - - pub fn add(&mut self, entry: Entry) -> bool { - let w: &mut dyn Write; - let mut id: String; - - id = entry.id.to_string(); - match &mut self.cache { - Some(v) => { - w = v.open(id); - }, - None => { - w = &mut self.default_cache; - }, - } - - id = entry.id.to_string(); - let o = SequencerEntry::new(entry, w); - if self.items.contains_key(&o.digest) { - return false; - } - self.items.insert(o.digest, o.into()); - match &mut self.cache { - Some(v) => { - v.close(id); - }, - None => { - }, - } - return true; - } - - pub fn add_from(&mut self, feed: Feed) -> i64 { - let mut c: i64; - - c = 0; - for v in feed.entries.iter() { - self.add(v.clone()); - c += 1; - } - c - } - - fn write_to(&mut self, w: impl Write) -> Result<usize, Error> { - let mut r: usize; - let mut feed = OutFeed::default(); - let mut entry: OutEntry; - let mut entries: Vec<OutEntry>; - let mut b: &str; - feed.set_id("urn:uuid:60a76c80-d399-11d9-b91C-0003939e0af6"); - feed.set_updated(Local::now().to_utc()); - - match self.metadata.apply(&mut feed) { - Err(_v) => { - return Err(Error::WriteError); - }, - Ok(_) => { - }, - } - - entries = Vec::new(); - r = 0; - for v in self { - b = std::str::from_utf8(v.as_slice()).unwrap(); - match OutEntry::from_str(b) { - Err(e) => { - println!("fromstrerr {:?}", e); - return Err(Error::CacheError); - }, - Ok(o) => { - entries.push(o); - }, - } - r += 1; - } - feed.set_entries(entries); - - match feed.write_to(w) { - Err(_v) => { - return Err(Error::WriteError); - }, - Ok(_) => { - }, - } - - Ok(r) - } -} - -impl<'a> Iterator for Sequencer<'a> { - type Item = Vec<u8>; - - fn next(&mut self) -> Option<Self::Item> { - let c: u64; - - if self.limit == 0 { - self.item_keys = Vec::new(); - for k in self.items.keys().sorted() { - self.item_keys.push(k.clone()); - self.limit += 1; - } - } - - if self.limit == 0 { - return None; - } - - if self.crsr == self.limit { - self.limit = 0; - self.crsr = 0; - return None; - } - - c = self.item_keys[self.crsr]; - self.crsr += 1; - return Some(self.items[&c].clone()); - } -} - -impl SequencerEntry { - pub fn new(entry: Entry, exporter: &mut dyn Write) -> SequencerEntry { - let mut have_date: bool; - let mut id_part: u32; - let mut o = SequencerEntry { - entry: entry, - digest: 0, - out: Vec::new(), - }; - - have_date = false; - match &o.entry.published { - Some(v) => { - id_part = v.timestamp() as u32; - o.digest = id_part as u64; - o.digest <<= 32; - have_date = true; - }, - None => { - }, - } - - if !have_date { - match &o.entry.updated { - Some(v) => { - id_part = v.timestamp() as u32; - o.digest = id_part as u64; - o.digest <<= 32; - have_date = true; - }, - None => { - }, - } - } - - let mut h = Sha512Hasher::default(); - o.hash(&mut h); - id_part = h.finish() as u32; - o.digest += id_part as u64; - o - } - - /// TODO: get size heuristics from already written values (either that or replace underlying - /// in-memory writer implementation with something that doesnt wrap on flush. - fn to_writer(&self, v: Vec<u8>) -> BufWriter<Vec<u8>> { - BufWriter::with_capacity(10241024, v) - } -} - -/// TODO: split out field translations to separate module -impl Into<Vec<u8>> for SequencerEntry { - fn into(self) -> Vec<u8> { - let mut out_entry: OutEntry; - let mut b: Vec<u8>; - let mut w: BufWriter<Vec<u8>>; - let o: &SequencerEntry; - - o = &self; - b = Vec::new(); - w = o.to_writer(b); - - out_entry = OutEntry::default(); - out_entry.set_id(self.entry.id); - out_entry.set_title(self.entry.title.unwrap().content); - - let mut d = FixedDateTime::parse_from_rfc2822(self.entry.published.unwrap().to_rfc2822().as_str()).unwrap(); - out_entry.set_published(Some(d.clone())); - - match self.entry.updated { - Some(v) => { - d = FixedDateTime::parse_from_rfc2822(v.to_rfc2822().as_str()).unwrap(); - out_entry.set_updated(d.clone()); - }, - None => {}, - } - - match self.entry.summary { - Some(v) => { - let text_out: OutText; - let summary_out_type: OutTextType; - let summary_subtype = String::from(v.content_type.subty().as_str()); - if summary_subtype.contains("xhtml") { - summary_out_type = OutTextType::Xhtml; - } else if summary_subtype.contains("html") { - summary_out_type = OutTextType::Html; - } else { - summary_out_type = OutTextType::Text; - } - text_out = OutText{ - value: v.content, - r#type: summary_out_type, - base: None, - lang: None, - }; - out_entry.set_summary(Some(text_out)); - }, - None => {}, - } - - match self.entry.content { - Some(v) => { - let mut content_out = OutContent::default(); - content_out.content_type = Some(String::from(v.content_type.as_str())); - match v.src { - Some(vv) => { - content_out.src = Some(vv.href); - }, - None => {}, - }; - match v.body { - Some(vv) => { - content_out.value = Some(vv); - }, - None => {}, - }; - out_entry.set_content(Some(content_out)); - }, - None => {}, - } - - for v in self.entry.authors { - let o = OutPerson{ - name: v.name, - uri: v.uri, - email: v.email, - }; - out_entry.authors.push(o); - } - - for v in self.entry.contributors { - let o = OutPerson{ - name: v.name, - uri: v.uri, - email: v.email, - }; - out_entry.contributors.push(o); - } - - for v in self.entry.categories { - let o = OutCategory { - term: v.term, - scheme: v.scheme, - label: v.label, - }; - out_entry.categories.push(o); - } - - w = out_entry.write_to(w).unwrap(); - b = Vec::from(w.buffer()); - b - } -} - -impl Hash for SequencerEntry { - fn hash<H: Hasher>(&self, h: &mut H) { - h.write(self.entry.id.as_bytes()); - } -} - -#[cfg(test)] -mod tests; diff --git a/src/main.rs b/src/main.rs @@ -0,0 +1,103 @@ +use std::path::absolute; +use std::path::PathBuf; +use std::process; +use std::io::stdout; +use std::str::from_utf8; + +use clap::Arg; +use clap::App; + +#[cfg(feature="logging")] +use log::debug; +use log::info; +use env_logger; + +use crier::Sequencer; +use crier::io::FeedGet; +use crier::mem::MemCache; +use crier::io::fs::FsFeed; +use crier::Error; + + +struct Config { + urls: Vec<String>, +} + +impl Config { + fn new(urls: Vec<String>) -> Config { + Config { + urls: urls, + } + } +} + +fn parse() -> Config { + let m = App::new("crier") + .version(env!("CARGO_PKG_VERSION")) + .author(env!("CARGO_PKG_AUTHORS")) + .arg(Arg::with_name("URLS") + .multiple(true) + .help("list of uris to merge")) + .get_matches(); + + Config::new(m.values_of("URLS").unwrap().map(|v| String::from(v)).collect()) +} + +fn add_feed(seq: &mut Sequencer, getter: impl FeedGet, uri: String) -> Result<i64, Error> { + match getter.get(uri.as_str(), None) { + Ok(v) => { + let r = seq.add_from(v); + info!("got {} results from {}", r, uri); + return Ok(r); + }, + Err(e) => { + return Err(Error::ParseError); + }, + }; +} + +fn process_entry(seq: &mut Sequencer, uri: String) -> Result<(), Error> { + let v: PathBuf; + let fp: String; + let fs = FsFeed{}; + + debug!("processing {}", uri); + match absolute(uri) { + Ok(r) => { + fp = String::from(r.to_str().unwrap()); + }, + Err(e) => { + return Err(Error::ParseError); + } + }; + + match add_feed(seq, fs, fp) { + Ok(r) => { + return Ok(()); + }, + Err(e) => { + return Err(e); + }, + } +} + +fn main() { + let cfg = parse(); + let mut cache = MemCache::new(); + let mut seq = Sequencer::new().with_cache(&mut cache); + + seq.set_title("my new feed"); + seq.set_author("Foo Bar"); + + +#[cfg(feature = "logging")] + env_logger::init(); + + debug!("config has {} uris", cfg.urls.len()); + + for v in cfg.urls { + process_entry(&mut seq, v).unwrap_or_else(|e| process::exit(1)); + } + + seq.write_to(stdout()).unwrap(); +} diff --git a/src/meta.rs b/src/meta.rs @@ -1,48 +0,0 @@ -use atom_syndication::Person; -use atom_syndication::Feed; -use uuid::Uuid; - -#[derive(Debug)] -pub enum Error { - IncompleteFeedMetadata, -} - - -pub struct FeedMetadata { - pub author: Person, - pub title: String, - pub id: String, - incomplete: bool, -} - -impl Default for FeedMetadata { - fn default() -> FeedMetadata { - FeedMetadata{ - author: Person{ - name: "?".to_string(), - email: Some("?".to_string()), - uri: Some("?".to_string()), - }, - title: String::from("?"), - id: Uuid::new_v4().to_string(), - incomplete: true, - } - } -} - -impl FeedMetadata { - pub fn force(&mut self) { - self.incomplete = false; - } - - pub fn apply(&self, feed: &mut Feed) -> Result<(), Error> { - if self.incomplete { - return Err(Error::IncompleteFeedMetadata); - } - let mut persons = Vec::<Person>::new(); - persons.push(self.author.clone()); - feed.set_authors(persons); - feed.set_title(self.title.clone()); - Ok(()) - } -} diff --git a/src/tests.rs b/src/tests.rs @@ -1,251 +0,0 @@ -use std::clone::Clone; -use std::fs::File; -use std::io::{SeekFrom, Seek, Read}; -use std::str; - -use feed_rs::model::Entry; -use feed_rs::model::Text; -use mediatype::MediaTypeBuf; -use chrono::DateTime; -use tempfile::NamedTempFile; -use tempfile::tempdir; -use atom_syndication::Entry as OutEntry; -use quick_xml::Reader as XMLReader; -use quick_xml::events::Event as XMLEvent; - -use crate::Sequencer; -use crate::io::FeedGet; -use crate::io::fs::FsCache; - -#[cfg(feature = "fs")] -use crate::io::fs::FsFeed; - - -fn check_xml_title(xml: Vec<u8>, title: &str) { - let mut rxml = XMLReader::from_str(str::from_utf8(&xml).unwrap()); - let mut xmlbuf = Vec::new(); - let mut state = 0; - loop { - match rxml.read_event_into(&mut xmlbuf) { - Err(e) => panic!("cant read back xml: {:?}", e), - Ok(XMLEvent::Eof) => break, - Ok(XMLEvent::Start(v)) => { - match v.name().as_ref() { - b"title" => { - state = 1; - }, - _ => { - state = 0 - }, - } - }, - Ok(XMLEvent::End(v)) => { - state = 0; - }, - Ok(XMLEvent::Text(v)) => { - if state > 0 { - assert_eq!(v.unescape().unwrap(), title); - } - }, - _ => (), - } - } -} - -#[test] -fn test_entry_guard() { - let mut r: bool; - let mut seq = Sequencer::new(); - let mut src = Entry::default(); - let mut s: String; - - src.id = String::from("foo"); - s = String::from("inky"); - src.title = Some(Text{ - content_type: MediaTypeBuf::from_string(String::from("text/plain")).unwrap(), - src: Some(s.clone()), - content: s, - - }); - - src.published = Some(DateTime::parse_from_rfc3339("2024-06-25T20:46:00+02:00").unwrap().into()); - r = seq.add(src); - assert!(r); - - let mut src_two = Entry::default(); - src_two.id = String::from("foo"); - s = String::from("pinky"); - src_two.title = Some(Text{ - content_type: MediaTypeBuf::from_string(String::from("text/plain")).unwrap(), - src: Some(s.clone()), - content: s, - - }); - src_two.published = Some(DateTime::parse_from_rfc3339("2024-06-25T20:46:00+02:00").unwrap().into()); - r = seq.add(src_two); - assert!(!r); - - let mut src_three = Entry::default(); - src_three.id = String::from("foo"); - s = String::from("blinky"); - src_three.title = Some(Text{ - content_type: MediaTypeBuf::from_string(String::from("text/plain")).unwrap(), - src: Some(s.clone()), - content: s, - - }); - - src_three.published = Some(DateTime::parse_from_rfc3339("2024-06-25T20:46:00+03:00").unwrap().into()); - r = seq.add(src_three); - assert!(r); -} - -#[test] -#[cfg(feature = "fs")] -fn test_feed_get() { - let r: bool; - let fs = FsFeed{}; - let feed = fs.get("testdata/test.atom.xml", None).unwrap(); - let mut seq = Sequencer::new(); - r = seq.add(feed.entries.get(0).unwrap().clone()); - assert!(r); -} - -#[test] -#[cfg(feature = "fs")] -fn test_feed_all() { - let r: i64; - let fs = FsFeed{}; - let feed = fs.get("testdata/test.atom.xml", None).unwrap(); - let mut seq = Sequencer::new(); - r = seq.add_from(feed); - assert_eq!(r, 16); -} - -#[test] -#[cfg(feature = "fs")] -fn test_feed_mix() { - let mut r: i64; - let fs = FsFeed{}; - let mut feed = fs.get("testdata/test.atom.xml", None).unwrap(); - let mut seq = Sequencer::new(); - r = seq.add_from(feed); - assert_eq!(r, 16); - feed = fs.get("testdata/test2.xml", None).unwrap(); - r = seq.add_from(feed); - assert_eq!(r, 10); - assert_eq!(seq.by_ref().count(), 26); - assert_eq!(seq.count(), 26); -} - -#[test] -#[cfg(feature = "fs")] -fn test_feed_write() { - let r: usize; - let fs = FsFeed{}; - let f: NamedTempFile; - let mut fr: File; - - let feed = fs.get("testdata/test.atom.xml", None).unwrap(); - let mut seq = Sequencer::new(); - seq.add_from(feed); - f = NamedTempFile::new().unwrap(); - fr = f.reopen().unwrap(); - r = seq.write_to(f).unwrap(); - assert_eq!(r, 16); - assert_eq!(fr.metadata().unwrap().len(), 519536); -} - -#[test] -#[cfg(feature = "fs")] -fn test_feed_write_extcache() { - let r: usize; - let fs = FsFeed{}; - let f: NamedTempFile; - let fr: File; - let mut cache: FsCache; - - let d = tempdir().unwrap(); - cache = FsCache::new(d.into_path()); - - let feed = fs.get("testdata/test.atom.xml", None).unwrap(); - let mut seq = Sequencer::new(); - seq = seq.with_cache(&mut cache); - - seq.add_from(feed); - f = NamedTempFile::new().unwrap(); - fr = f.reopen().unwrap(); - r = seq.write_to(f).unwrap(); - - assert_eq!(r, 16); - assert_eq!(fr.metadata().unwrap().len(), 519536); -} - -#[test] -#[cfg(feature = "fs")] -fn test_sequence_order() { - let mut seq = Sequencer::new(); - let mut entry: Entry; - let mut s: String; - let mut r: Vec<u8>; - - entry = Entry::default(); - entry.id = String::from("y"); - s = String::from("inky"); - entry.title = Some(Text{ - content_type: MediaTypeBuf::from_string(String::from("text/plain")).unwrap(), - src: Some(s.clone()), - content: s, - - }); - entry.published = Some(DateTime::parse_from_rfc3339("2024-06-25T20:46:00+02:00").unwrap().into()); - seq.add(entry); - - - entry = Entry::default(); - entry.id = String::from("b"); - s = String::from("pinky"); - entry.title = Some(Text{ - content_type: MediaTypeBuf::from_string(String::from("text/plain")).unwrap(), - src: Some(s.clone()), - content: s, - - }); - entry.published = Some(DateTime::parse_from_rfc3339("2023-06-25T20:46:00+02:00").unwrap().into()); - seq.add(entry); - - entry = Entry::default(); - entry.id = String::from("d"); - s = String::from("blinky"); - entry.title = Some(Text{ - content_type: MediaTypeBuf::from_string(String::from("text/plain")).unwrap(), - src: Some(s.clone()), - content: s, - - }); - entry.published = Some(DateTime::parse_from_rfc3339("2024-06-25T20:46:00+02:00").unwrap().into()); - seq.add(entry); - - entry = Entry::default(); - entry.id = String::from("a"); - s = String::from("clyde"); - entry.title = Some(Text{ - content_type: MediaTypeBuf::from_string(String::from("text/plain")).unwrap(), - src: Some(s.clone()), - content: s, - - }); - entry.published = Some(DateTime::parse_from_rfc3339("2024-06-25T20:46:00+02:00").unwrap().into()); - seq.add(entry); - - - // TODO find value where sort digest is reverse of lexical id - r = seq.next().unwrap(); - check_xml_title(r, "pinky"); - r = seq.next().unwrap(); - check_xml_title(r, "blinky"); - r = seq.next().unwrap(); - check_xml_title(r, "inky"); - r = seq.next().unwrap(); - check_xml_title(r, "clyde"); -}