diff --git a/Cargo.lock b/Cargo.lock index 29193e1..052bce6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "addr2line" @@ -61,15 +61,21 @@ dependencies = [ [[package]] name = "allocator-api2" -version = "0.2.18" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" +checksum = "45862d1c77f2228b9e10bc609d5bc203d86ebc9b87ad8d5d5167a6c9abf739d9" [[package]] name = "anstyle" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8365de52b16c035ff4fcafe0092ba9390540e3e352870ac09933bebcaa2c8c56" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anyhow" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" [[package]] name = "arc-swap" @@ -101,9 +107,9 @@ dependencies = [ [[package]] name = "atom_syndication" -version = "0.12.4" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a3a5ed3201df5658d1aa45060c5a57dc9dba8a8ada20d696d67cb0c479ee043" +checksum = "3ee79fb83c725eae67b55218870813d2fc39fd85e4f1583848ef9f4f823cfe7c" dependencies = [ "chrono", "derive_builder", @@ -183,9 +189,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c" +checksum = "1a68f1f47cdf0ec8ee4b941b2eee2a80cb796db73118c0dd09ac63fbe405be22" dependencies = [ "memchr", "serde", @@ -197,6 +203,12 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "225eb82ce9e70dcc0cfa6e404d0f353326b6e163bf500ec4711cec317d11935c" +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + [[package]] name = "byteorder" version = "1.5.0" @@ -211,9 +223,9 @@ checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" [[package]] name = "cached" -version = "0.51.4" +version = "0.54.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0feb64151eed3da6107fddd2d717a6ca4b9dbd74e43784c55c841d1abfe5a295" +checksum = "9718806c4a2fe9e8a56fd736f97b340dd10ed1be8ed733ed50449f351dc33cae" dependencies = [ "ahash", "async-trait", @@ -221,17 +233,17 @@ dependencies = [ "cached_proc_macro_types", "futures", "hashbrown 0.14.5", - "instant", "once_cell", "thiserror", "tokio", + "web-time", ] [[package]] name = "cached_proc_macro" -version = "0.21.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "771aa57f3b17da6c8bcacb187bb9ec9bc81c8160e72342e67c329e0e1651a669" +checksum = "2f42a145ed2d10dce2191e1dcf30cfccfea9026660e143662ba5eec4017d5daa" dependencies = [ "darling", "proc-macro2", @@ -247,9 +259,9 @@ checksum = "ade8366b8bd5ba243f0a58f036cc0ca8a2f069cff1a2351ef1cac6b083e16fc0" [[package]] name = "cc" -version = "1.1.31" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f" +checksum = "fd9de9f2205d5ef3fd67e685b0df337994ddd4495e2a28d185500d0e1edfea47" dependencies = [ "shlex", ] @@ -271,9 +283,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.20" +version = "4.5.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" +checksum = "fb3b4b9e5a7c7514dfa52869339ee98b3156b0bfb4e8a77c4ff4babb64b1604f" dependencies = [ "clap_builder", "clap_derive", @@ -281,9 +293,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.20" +version = "4.5.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" +checksum = "b17a95aa67cc7b5ebd32aa5370189aa0d79069ef1c64ce893bd30fb24bff20ec" dependencies = [ "anstyle", "clap_lex", @@ -303,9 +315,21 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +checksum = "afb84c814227b90d6895e01398aee0d8033c00e7466aca416fb6a8e0eb19d8a7" + +[[package]] +name = "common-words-all" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84a6ff47eb813c9e315610ceca0ddd247827e22f2cdadc4189e4676a81470c77" +dependencies = [ + "anyhow", + "csv", + "glob", + "serde", +] [[package]] name = "cookie" @@ -344,9 +368,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +checksum = "0ca741a962e1b0bff6d724a1a0958b686406e853bb14061f218562e1896f95e6" dependencies = [ "libc", ] @@ -370,6 +394,27 @@ dependencies = [ "typenum", ] +[[package]] +name = "csv" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + [[package]] name = "darling" version = "0.20.10" @@ -463,13 +508,24 @@ dependencies = [ [[package]] name = "diligent-date-parser" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6cf7fe294274a222363f84bcb63cdea762979a0443b4cf1f4f8fd17c86b1182" +checksum = "c8ede7d79366f419921e2e2f67889c12125726692a313bffb474bd5f37a581e9" dependencies = [ "chrono", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "dotenvy" version = "0.15.7" @@ -516,9 +572,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" +checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" [[package]] name = "fnv" @@ -579,9 +635,9 @@ checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-lite" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f1fa2f9765705486b33fd2acf1577f8ec449c2ba1f318ae5447697b7c08d210" +checksum = "cef40d21ae2c515b51041df9ed313ed21e572df340ea58a922a0aefe7e8891a1" dependencies = [ "fastrand", "futures-core", @@ -642,6 +698,12 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + [[package]] name = "globset" version = "0.4.15" @@ -686,9 +748,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.0" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" [[package]] name = "heck" @@ -788,6 +850,124 @@ dependencies = [ "tokio-rustls", ] +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -796,12 +976,23 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "0.5.0" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", ] [[package]] @@ -811,16 +1002,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.15.0", -] - -[[package]] -name = "instant" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" -dependencies = [ - "cfg-if", + "hashbrown 0.15.1", ] [[package]] @@ -847,10 +1029,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] -name = "libc" -version = "0.2.161" +name = "js-sys" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" +checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.164" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "433bfe06b8c75da9b2e3fbea6e5329ff87748f0b144ef75306e674c3f6f7c13f" [[package]] name = "libflate" @@ -892,6 +1083,12 @@ dependencies = [ "rand_chacha", ] +[[package]] +name = "litemap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" + [[package]] name = "lock_api" version = "0.4.12" @@ -1107,9 +1304,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quick-xml" -version = "0.36.2" +version = "0.37.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe" +checksum = "f22f29bdff3987b4d8632ef95fd6424ec7e4e0a57e2f4fc63e489e75357f6a03" dependencies = [ "encoding_rs", "memchr", @@ -1169,6 +1366,7 @@ dependencies = [ "build_html", "cached", "clap", + "common-words-all", "cookie", "dotenvy", "fastrand", @@ -1212,9 +1410,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", @@ -1292,9 +1490,9 @@ checksum = "afab94fb28594581f62d981211a9a4d53cc8130bbcbbb89a0440d9b8e81a7746" [[package]] name = "rss" -version = "2.0.9" +version = "2.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27e92048f840d98c6d6dd870af9101610ea9ff413f11f1bcebf4f4c31d96d957" +checksum = "554a62b3dd5450fcbb0435b3db809f9dd3c6e9f5726172408f7ad3b57ed59057" dependencies = [ "atom_syndication", "derive_builder", @@ -1351,9 +1549,9 @@ checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" [[package]] name = "rustix" -version = "0.38.38" +version = "0.38.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa260229e6538e52293eeb577aabd09945a09d6d9cc0fc550ed7529056c2e32a" +checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6" dependencies = [ "bitflags", "errno", @@ -1434,9 +1632,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01227be5826fa0690321a2ba6c5cd57a19cf3f6a09e76973b58e61de6ab9d1c1" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" dependencies = [ "windows-sys 0.59.0", ] @@ -1494,9 +1692,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.12.0" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6" +checksum = "fa39c7303dc58b5543c94d22c1766b0d31f2ee58306363ea622b10bbc075eaa2" dependencies = [ "core-foundation-sys", "libc", @@ -1504,18 +1702,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.214" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5" +checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.214" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" +checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" dependencies = [ "proc-macro2", "quote", @@ -1524,9 +1722,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.132" +version = "1.0.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" +checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" dependencies = [ "itoa", "memchr", @@ -1536,13 +1734,12 @@ dependencies = [ [[package]] name = "serde_json_path" -version = "0.6.7" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bc0207b6351893eafa1e39aa9aea452abb6425ca7b02dd64faf29109e7a33ba" +checksum = "e176fbf9bd62f75c2d8be33207fa13af2f800a506635e89759e46f934c520f4d" dependencies = [ "inventory", "nom", - "once_cell", "regex", "serde", "serde_json", @@ -1553,12 +1750,11 @@ dependencies = [ [[package]] name = "serde_json_path_core" -version = "0.1.6" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3d64fe53ce1aaa31bea2b2b46d3b6ab6a37e61854bedcbd9f174e188f3f7d79" +checksum = "ea3bfd54a421bec8328aefede43ac9f18c8c7ded3b2afc8addd44b4813d99fd0" dependencies = [ "inventory", - "once_cell", "serde", "serde_json", "thiserror", @@ -1566,21 +1762,20 @@ dependencies = [ [[package]] name = "serde_json_path_macros" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a31e8177a443fd3e94917f12946ae7891dfb656e6d4c5e79b8c5d202fbcb723" +checksum = "ee05bac728cc5232af5c23896b34fbdd17cf0bb0c113440588aeeb1b57c6ba1f" dependencies = [ "inventory", - "once_cell", "serde_json_path_core", "serde_json_path_macros_internal", ] [[package]] name = "serde_json_path_macros_internal" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75dde5a1d2ed78dfc411fc45592f72d3694436524d3353683ecb3d22009731dc" +checksum = "aafbefbe175fa9bf03ca83ef89beecff7d2a95aaacd5732325b90ac8c3bd7b90" dependencies = [ "proc-macro2", "quote", @@ -1666,6 +1861,12 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "strsim" version = "0.11.1" @@ -1674,9 +1875,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "syn" -version = "2.0.86" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89275301d38033efb81a6e60e3497e734dfcc62571f2854bf4b16690398824c" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", @@ -1684,10 +1885,21 @@ dependencies = [ ] [[package]] -name = "tempfile" -version = "3.13.0" +name = "synstructure" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tempfile" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" dependencies = [ "cfg-if", "fastrand", @@ -1707,18 +1919,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.65" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d11abd9594d9b38965ef50805c5e469ca9cc6f197f883f717e0269a3057b3d5" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.65" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae71770322cbd277e69d762a16c444af02aa0575ac0d174f0b9562d3b37f8602" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", @@ -1759,25 +1971,20 @@ dependencies = [ ] [[package]] -name = "tinyvec" -version = "1.8.0" +name = "tinystr" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" dependencies = [ - "tinyvec_macros", + "displaydoc", + "zerovec", ] -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - [[package]] name = "tokio" -version = "1.41.0" +version = "1.41.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145f3413504347a2be84393cc8a7d2fb4d863b375909ea59f2158261aa258bbb" +checksum = "22cfb5bee7a6a52939ca9224d6ac897bb669134078daa8735560897f69de4d33" dependencies = [ "backtrace", "bytes", @@ -1902,27 +2109,12 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df" -[[package]] -name = "unicode-bidi" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" - [[package]] name = "unicode-ident" version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" -[[package]] -name = "unicode-normalization" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" -dependencies = [ - "tinyvec", -] - [[package]] name = "unsafe-libyaml" version = "0.2.11" @@ -1937,15 +2129,27 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.2" +version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" dependencies = [ "form_urlencoded", "idna", "percent-encoding", ] +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "uuid" version = "1.11.0" @@ -1995,6 +2199,71 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" +dependencies = [ + "cfg-if", + "once_cell", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "winapi-util" version = "0.1.9" @@ -2095,6 +2364,42 @@ dependencies = [ "memchr", ] +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + +[[package]] +name = "yoke" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.7.35" @@ -2115,3 +2420,46 @@ dependencies = [ "quote", "syn", ] + +[[package]] +name = "zerofrom" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/Cargo.toml b/Cargo.toml index 9023c4e..7d6cdae 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ default-run = "redsunlib" [dependencies] rinja = { version = "0.3.4", default-features = false } -cached = { version = "0.51.3", features = ["async"] } +cached = { version = "0.54.0", features = ["async"] } clap = { version = "4.4.11", default-features = false, features = [ "std", "env", @@ -23,11 +23,10 @@ regex = "1.10.2" serde = { version = "1.0.193", features = ["derive"] } cookie = "0.18.0" futures-lite = "2.2.0" -hyper = { version = "0.14.28", features = ["full"] } -hyper-rustls = { version = "0.24.2", features = [ "http2" ] } +hyper = { version = "0.14.31", features = ["full"] } percent-encoding = "2.3.1" route-recognizer = "0.3.1" -serde_json = "1.0.108" +serde_json = "1.0.133" tokio = { version = "1.35.1", features = ["full"] } time = { version = "0.3.31", features = ["local-offset"] } url = "2.5.0" @@ -46,8 +45,10 @@ pretty_env_logger = "0.5.0" dotenvy = "0.15.7" rss = "2.0.7" arc-swap = "1.7.1" -serde_json_path = "0.6.7" +serde_json_path = "0.7.1" async-recursion = "1.1.1" +common-words-all = { version = "0.0.2", default-features = false, features = ["english", "one"] } +hyper-rustls = { version = "0.24.2", features = [ "http2" ] } [dev-dependencies] diff --git a/redlib.container b/redlib.container new file mode 100644 index 0000000..e66051e --- /dev/null +++ b/redlib.container @@ -0,0 +1,16 @@ +[Install] +WantedBy=default.target + +[Container] +AutoUpdate=registry +ContainerName=redlib +DropCapability=ALL +EnvironmentFile=.env +HealthCmd=["wget","--spider","-q","--tries=1","http://localhost:8080/settings"] +HealthInterval=5m +HealthTimeout=3s +Image=quay.io/redlib/redlib:latest +NoNewPrivileges=true +PublishPort=8080:8080 +ReadOnly=true +User=nobody diff --git a/src/client.rs b/src/client.rs index 3d20db0..248fc88 100644 --- a/src/client.rs +++ b/src/client.rs @@ -231,27 +231,37 @@ fn request(method: &'static Method, path: String, redirect: bool, quarantine: bo // Build request to Reddit. When making a GET, request gzip compression. // (Reddit doesn't do brotli yet.) - let builder = Request::builder() - .method(method) - .uri(&url) - .header("User-Agent", user_agent) - .header("Client-Vendor-Id", vendor_id) - .header("X-Reddit-Device-Id", device_id) - .header("x-reddit-loid", loid) - .header("Host", host) - .header("Authorization", &format!("Bearer {token}")) - .header("Accept-Encoding", if method == Method::GET { "gzip" } else { "identity" }) - .header("Accept-Language", "en-US,en;q=0.5") - .header("Connection", "keep-alive") - .header( + let mut headers = vec![ + ("User-Agent", user_agent), + ("Client-Vendor-Id", vendor_id), + ("X-Reddit-Device-Id", device_id), + ("x-reddit-loid", loid), + ("Host", host.to_string()), + ("Authorization", format!("Bearer {token}")), + ("Accept-Encoding", if method == Method::GET { "gzip".into() } else { "identity".into() }), + ( "Cookie", if quarantine { - "_options=%7B%22pref_quarantine_optin%22%3A%20true%2C%20%22pref_gated_sr_optin%22%3A%20true%7D" + "_options=%7B%22pref_quarantine_optin%22%3A%20true%2C%20%22pref_gated_sr_optin%22%3A%20true%7D".into() } else { - "" + "".into() }, - ) - .body(Body::empty()); + ), + ("X-Reddit-Width", fastrand::u32(300..500).to_string()), + ("X-Reddit-DPR", "2".to_owned()), + ("Device-Name", format!("Android {}", fastrand::u8(9..=14))), + ]; + + // shuffle headers: https://github.com/redlib-org/redlib/issues/324 + fastrand::shuffle(&mut headers); + + let mut builder = Request::builder().method(method).uri(&url); + + for (key, value) in headers { + builder = builder.header(key, value); + } + + let builder = builder.body(Body::empty()); async move { match builder { diff --git a/src/main.rs b/src/main.rs index a66f734..101b2b1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -248,6 +248,7 @@ async fn main() { .get(|_| resource(include_str!("../static/check_update.js"), "text/javascript", false).boxed()); app.at("/commits.json").get(|_| async move { proxy_commit_info().await }.boxed()); + app.at("/instances.json").get(|_| async move { proxy_instances().await }.boxed()); // FFmpeg app @@ -448,3 +449,22 @@ async fn fetch_commit_info() -> String { hyper::body::to_bytes(resp).await.expect("Failed to read body").iter().copied().map(|x| x as char).collect() } + +pub async fn proxy_instances() -> Result, String> { + Ok( + Response::builder() + .status(200) + .header("content-type", "application/json") + .body(Body::from(fetch_instances().await)) + .unwrap_or_default(), + ) +} + +#[cached(time = 600)] +async fn fetch_instances() -> String { + let uri = Uri::from_str("https://raw.githubusercontent.com/redlib-org/redlib-instances/refs/heads/main/instances.json").expect("Invalid URI"); + + let resp: Body = CLIENT.get(uri).await.expect("Failed to request GitHub").into_body(); + + hyper::body::to_bytes(resp).await.expect("Failed to read body").iter().copied().map(|x| x as char).collect() +} diff --git a/src/scraper/main.rs b/src/scraper/main.rs index a1ebe35..acbe47e 100644 --- a/src/scraper/main.rs +++ b/src/scraper/main.rs @@ -1,6 +1,7 @@ -use std::{fmt::Display, io::Write}; +use std::{collections::HashMap, fmt::Display, io::Write}; use clap::{Parser, ValueEnum}; +use common_words_all::{get_top, Language, NgramSize}; use redsunlib::utils::Post; #[derive(Parser)] @@ -10,9 +11,6 @@ struct Cli { #[arg(short = 's', long = "sub")] sub: String, - #[arg(short = 'c', long = "count")] - count: usize, - #[arg(long = "sort")] sort: SortOrder, @@ -50,26 +48,85 @@ enum Format { #[tokio::main] async fn main() { + pretty_env_logger::init(); let cli = Cli::parse(); - let (sub, final_count, sort, format, output) = (cli.sub, cli.count, cli.sort, cli.format, cli.output); + let (sub, sort, format, output) = (cli.sub, cli.sort, cli.format, cli.output); let initial = format!("/r/{sub}/{sort}.json?&raw_json=1"); - let (mut posts, mut after) = Post::fetch(&initial, false).await.unwrap(); - while posts.len() < final_count { + let (posts, mut after) = Post::fetch(&initial, false).await.unwrap(); + let mut hashmap = HashMap::new(); + hashmap.extend(posts.into_iter().map(|post| (post.id.clone(), post))); + loop { print!("\r"); let path = format!("/r/{sub}/{sort}.json?sort={sort}&t=&after={after}&raw_json=1"); let (new_posts, new_after) = Post::fetch(&path, false).await.unwrap(); - posts.extend(new_posts); + let old_len = hashmap.len(); + // convert to hashmap and extend hashmap + let new_posts = new_posts.into_iter().map(|post| (post.id.clone(), post)).collect::>(); + let len = new_posts.len(); + hashmap.extend(new_posts); + if hashmap.len() - old_len < 3 { + break; + } + + let x = hashmap.len() - old_len; after = new_after; // Print number of posts fetched - print!("Fetched {} posts", posts.len()); + print!("Fetched {len} posts (+{x})",); std::io::stdout().flush().unwrap(); } + println!("\n\n"); + // additionally search if final count not reached + for word in get_top(Language::English, 10_000, NgramSize::One) { + let mut retrieved_posts_from_search = 0; + let initial = format!("/r/{sub}/search.json?q={word}&restrict_sr=on&include_over_18=on&raw_json=1&sort={sort}"); + println!("Grabbing posts with word {word}."); + let (posts, mut after) = Post::fetch(&initial, false).await.unwrap(); + hashmap.extend(posts.into_iter().map(|post| (post.id.clone(), post))); + 'search: loop { + let path = format!("/r/{sub}/search.json?q={word}&restrict_sr=on&include_over_18=on&raw_json=1&sort={sort}&after={after}"); + let (new_posts, new_after) = Post::fetch(&path, false).await.unwrap(); + if new_posts.is_empty() || new_after.is_empty() { + println!("No more posts for word {word}"); + break 'search; + } + retrieved_posts_from_search += new_posts.len(); + let old_len = hashmap.len(); + let new_posts = new_posts.into_iter().map(|post| (post.id.clone(), post)).collect::>(); + let len = new_posts.len(); + hashmap.extend(new_posts); + let delta = hashmap.len() - old_len; + after = new_after; + // Print number of posts fetched + println!("Fetched {len} posts (+{delta})",); + + if retrieved_posts_from_search > 1000 { + println!("Reached 1000 posts from search"); + break 'search; + } + } + // Need to save incrementally. atomic save + move + let tmp_file = output.clone().unwrap_or_else(|| format!("{sub}.json.tmp")); + let perm_file = output.clone().unwrap_or_else(|| format!("{sub}.json")); + write_posts(&hashmap.values().collect(), tmp_file.clone()); + // move file + std::fs::rename(tmp_file, perm_file).unwrap(); + } + + println!("\n\n"); + + println!("Size of hashmap: {}", hashmap.len()); + + let posts: Vec<&Post> = hashmap.values().collect(); match format { Format::Json => { let filename: String = output.unwrap_or_else(|| format!("{sub}.json")); - let json = serde_json::to_string(&posts).unwrap(); - std::fs::write(filename, json).unwrap(); + write_posts(&posts, filename); } } } + +fn write_posts(posts: &Vec<&Post>, filename: String) { + let json = serde_json::to_string(&posts).unwrap(); + std::fs::write(filename, json).unwrap(); +} diff --git a/src/utils.rs b/src/utils.rs index ad87d28..5b72a89 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -507,7 +507,7 @@ impl std::ops::Deref for Awards { impl std::fmt::Display for Awards { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - self.iter().fold(Ok(()), |result, award| result.and_then(|()| writeln!(f, "{award}"))) + self.iter().try_fold((), |_, award| writeln!(f, "{award}")) } } diff --git a/static/check_update.js b/static/check_update.js index 0bf42c8..fc9cd61 100644 --- a/static/check_update.js +++ b/static/check_update.js @@ -18,8 +18,10 @@ async function checkInstanceUpdateStatus() { statusMessage = '✅ Instance is up to date.'; } else if (commitIndex > 0) { statusMessage = `⚠️ This instance is not up to date and is ${commitIndex} commits old. Test and confirm on an up-to-date instance before reporting.`; + document.getElementById('error-318').remove(); } else { statusMessage = `⚠️ This instance is not up to date and is at least ${commitHashes.length} commits old. Test and confirm on an up-to-date instance before reporting.`; + document.getElementById('error-318').remove(); } } else { statusMessage = '⚠️ Unable to fetch commit information.'; @@ -32,4 +34,22 @@ async function checkInstanceUpdateStatus() { } } +async function checkOtherInstances() { + try { + const response = await fetch('/instances.json'); + const data = await response.json(); + const randomInstance = data.instances[Math.floor(Math.random() * data.instances.length)]; + const instanceUrl = randomInstance.url; + // Set the href of the tag to the instance URL with path included + document.getElementById('random-instance').href = instanceUrl + window.location.pathname; + document.getElementById('random-instance').innerText = "Visit Random Instance"; + } catch (error) { + console.error('Error fetching instances:', error); + document.getElementById('update-status').innerText = '⚠️ Error checking update status.'; + } +} + +// Set the target URL when the page loads +window.addEventListener('load', checkOtherInstances); + checkInstanceUpdateStatus(); diff --git a/templates/error.html b/templates/error.html index f892d14..3057203 100644 --- a/templates/error.html +++ b/templates/error.html @@ -7,6 +7,8 @@

Reddit Status


+
+


@@ -15,6 +17,8 @@

!! Do NOT open an issue on the redlib repository with a redsunlib specific issue !!


+

If you're getting a "Failed to parse page JSON data" error, please check #318

+

Head back home?

{% endblock %}