From a078c99ea20031059b63bf790bde8a7c40672766 Mon Sep 17 00:00:00 2001 From: lou lecrivain Date: Mon, 21 Sep 2026 17:24:52 +0200 Subject: [PATCH 01/11] add sample test data --- .../nrtmv3_ripe_mixed_encoding_sample.txt | 915 ++++++++++++++++++ 1 file changed, 915 insertions(+) create mode 100644 src/tests/nrtmv3_ripe_mixed_encoding_sample.txt diff --git a/src/tests/nrtmv3_ripe_mixed_encoding_sample.txt b/src/tests/nrtmv3_ripe_mixed_encoding_sample.txt new file mode 100644 index 0000000..17d894b --- /dev/null +++ b/src/tests/nrtmv3_ripe_mixed_encoding_sample.txt @@ -0,0 +1,915 @@ +% The RIPE Database is subject to Terms and Conditions. +% See https://docs.db.ripe.net/terms-conditions.html + +%START Version: 3 RIPE 65934907-65941039 + +DEL 65934907 + +inet6num: 2a01:4f8:173:171a::/64 +netname: HOS-2063226 +descr: HOS-2063226 +country: DE +admin-c: DUMY-RIPE +tech-c: DUMY-RIPE +status: ASSIGNED +notify: ripe-mntner@hetzner.com +mnt-by: HOS-GUN +created: 2024-10-07T11:03:23Z +last-modified: 2024-10-07T11:03:23Z +source: RIPE +remarks: **************************** +remarks: * THIS OBJECT IS MODIFIED +remarks: * Please note that all data that is generally regarded as personal +remarks: * data has been removed from this object. +remarks: * To view the original object, please query the RIPE Database at: +remarks: * http://www.ripe.net/whois +remarks: **************************** + +DEL 65934909 + +inetnum: 89.19.12.0 - 89.19.12.15 +netname: Elif-Yelkenci-Net +descr: Elif Yelkenci +country: TR +admin-c: DUMY-RIPE +tech-c: DUMY-RIPE +remarks: Please abuse mail to info@ylknc.com +status: ASSIGNED PA +mnt-by: CIZGI-MNT +created: 2026-09-14T21:00:09Z +last-modified: 2026-09-14T21:00:09Z +source: RIPE +remarks: **************************** +remarks: * THIS OBJECT IS MODIFIED +remarks: * Please note that all data that is generally regarded as personal +remarks: * data has been removed from this object. +remarks: * To view the original object, please query the RIPE Database at: +remarks: * http://www.ripe.net/whois +remarks: **************************** + +ADD 65934910 + +inetnum: 89.19.12.0 - 89.19.12.15 +netname: Elif-Yelkenci-Net +descr: Elif Yelkenci +country: TR +admin-c: DUMY-RIPE +tech-c: DUMY-RIPE +remarks: Please abuse mail to info@ylknc.com +status: ASSIGNED PA +mnt-by: CIZGI-MNT +created: 2026-09-14T22:00:16Z +last-modified: 2026-09-14T22:00:16Z +source: RIPE +remarks: **************************** +remarks: * THIS OBJECT IS MODIFIED +remarks: * Please note that all data that is generally regarded as personal +remarks: * data has been removed from this object. +remarks: * To view the original object, please query the RIPE Database at: +remarks: * http://www.ripe.net/whois +remarks: **************************** + +DEL 65934932 + +role: Abuse-C Role +address: Dummy address for AR80181-RIPE +e-mail: unread@ripe.net +abuse-mailbox: NOC@HYTRON.IO +nic-hdl: AR80181-RIPE +mnt-by: Hytronnetwork-MNT +created: 2026-06-11T09:00:31Z +last-modified: 2026-06-11T09:00:31Z +source: RIPE +remarks: **************************** +remarks: * THIS OBJECT IS MODIFIED +remarks: * Please note that all data that is generally regarded as personal +remarks: * data has been removed from this object. +remarks: * To view the original object, please query the RIPE Database at: +remarks: * http://www.ripe.net/whois +remarks: **************************** + +ADD 65934937 + +as-set: AS-VM-JE-CUS +remarks: ROUTE64.ORG / FREETRANSIT.DE CUSTOMERS +remarks: !!! PLEASE NOTE, THIS AS-SET IS AUTOMATICALLY GENERATED VIA REGISTRATION THROUGH ROUTE64.ORG!!! +remarks: !!! ALL NETWORKS (ASNS) HAVE ISSUED ROUTE64.ORG AS UPSTREAM PROVIDER FOR THEIR NETWORKS !!! +remarks: !!! ALL NETWORKS (ASNS) HAVE AN ACTIVE BGP SESSION WITH ROUTE64.ORG WITHIN THE LAST 7 DAYS !!! +tech-c: DUMY-RIPE +admin-c: DUMY-RIPE +mnt-by: vm_je +mnt-by: vm-je +mnt-by: route64-MNT +created: 2021-03-01T10:04:05Z +last-modified: 2026-09-14T22:04:21Z +source: RIPE +members: AS18710 +members: AS214844:AS-CSTUDNETZ +members: AS209828:AS-GENCBT +members: AS214260:AS-ITCOMMS +members: AS219261 +members: AS218917 +members: AS205591:AS-MAIN +members: AS198741 +members: AS204181:AS-DOWNSTREAM +members: AS212112 +members: AS205563 +members: AS138998 +members: AS199868:AS-ALL +members: AS-LHSNETWORKS +members: AS198016:AS-UPSTREAM +members: AS197438:AS-JBPRIV +members: AS215006 +members: AS215212:AS-NETNEXT +members: AS198767 +members: AS213472:AS-KOEHR +members: AS214556:AS-VEXURA +members: AS-NYALLO +members: AS212834:AS-DUREL +members: AS-CRIMPAC +members: AS208223 +members: AS208733:AS-FFVEC +members: AS199211 +members: AS218883 +members: AS207590 +members: AS197758:AS-NXLAYER +members: AS-TOZTECH +members: AS205413 +members: AS205305:AS-PANDEM +members: AS215486:AS-ALL +members: AS-TRUEWINTER +members: AS213416:AS-TISCH +members: AS211688:AS-WOLFFY +members: AS200455:AS-ALL +members: AS204880 +members: AS199950:AS-HJELMENTERPRISES +members: AS206058 +members: AS209185 +members: AS205548 +members: AS201379 +members: AS208487 +members: AS400941 +members: AS150790 +members: AS211626:AS-ASRO +members: AS215375 +members: AS201541 +members: AS203233 +members: AS210796:AS-BS +members: AS202322 +members: AS401748 +members: AS205893 +members: AS216191 +members: AS208589 +members: AS218822:AS-PMHONLY +members: AS201238 +members: AS214269 +members: AS208709 +members: AS-RAPIDZONE +members: AS401113 +members: AS-OXAR +members: AS216052:AS-ALL +members: AS214450 +members: AS-IMP-NET +members: AS216438:AS-HLNETWORKING +members: AS215931:AS-BARNA +members: AS210578:AS-MCPAL +members: AS215004:AS-BRNET +members: AS219421:AS-ALL +members: AS207873 +members: AS199183:AS-ESSINK +members: AS328949 +members: AS-IPV6-EDU-PL +members: AS208584 +members: AS213471 +members: AS198599 +members: AS-DAVOSYS +members: AS-XYZ +members: AS219051 +members: AS57027:AS-BASE +members: AS215459:AS-DOWNSTREAM +members: AS213430:AS-TRANSIT +members: AS219271:AS-MAIN +members: AS212691:AS-SET-ARONSTRECKER +members: AS201392:AS-TEAMINVERSELLC +members: AS210825:AS-TEIGTASCHERLFABRIK +members: AS209182 +members: AS206000:AS-SET +members: AS219453 +members: AS9423 +members: AS210732:AS-ECOWAN +members: AS198826 +members: AS203062 +members: AS209527:AS-DYNAMIX +members: AS208694:AS-TIEKOETTER +members: AS200116 +members: AS198120 +members: AS198408 +members: AS61953:AS-TRANSIT +members: AS214642:AS-SET +members: AS203453 +members: AS199674 +members: AS202886 +members: AS213297:AS-SET-WELTMEYER1 +members: AS200368:AS-ABNORMALFREQ +members: AS-LIGHTENING +members: AS214420:AS-CLNEXUS +members: AS-ISHRAK +members: AS219412 +members: AS213279 +members: AS198879 +members: AS205539 +members: AS218842 +members: AS207650 +members: AS199763:AS-PCELA +members: AS214223:AS-FPNET +members: AS402975:AS-DALINIET +members: AS214749:AS-PAW-INTERNAL +members: AS-SARUNETWORK +members: AS209745:AS-ONET +members: AS202751 +members: AS58626 +members: AS214549 +members: AS401593:AS-PEERS +members: AS207596:AS-WECLOUD +members: AS210957:AS-SITS +members: AS208301:AS-ANY +members: AS40929 +members: AS214508 +members: AS151669 +members: AS218890 +members: AS205621:AS-SELF +members: AS210106:AS-HUSKYHQ +members: AS203539 +members: AS197338 +members: AS218705 +members: AS51351 +members: AS205991 +members: AS201345 +members: AS219178:AS-BOGERTDEV +members: AS204463 +members: AS210074:AS-DOWNSTREAM +members: AS215561 +members: AS200879:AS-JCOLIDELES +members: AS203686:AS-ILH +members: AS205165 +members: AS208059 +members: AS402145 +members: AS202909 +members: AS136842 +members: AS207466 +members: AS216456 +members: AS210881 +members: AS219299 +members: AS211370 +members: AS401135 +members: AS197371 +members: AS213467:AS-PROADMIN +members: AS-CEDARCREEK +members: AS-KING +members: AS197601:AS-FDX-NET +members: AS-XIWEI +members: AS214344 +members: AS218804 +members: AS154408 +members: AS215664 +members: AS203868 +members: AS205336:AS-DEEPSPACECOW +members: AS214076:AS-ECKER +members: AS201705:AS-XLINET +members: AS214408 +members: AS402704 +members: AS219158 +members: AS397656 +members: AS209710:AS-SELF +members: AS219251 +members: AS-SADDLE +members: AS208679:AS-YIKAI-NET +members: AS213292:AS-OKKEL +members: AS-POLARIS +members: AS219013 +members: AS199854 +members: AS197736 +members: AS215228 +members: AS206115 +members: AS401703 +members: AS214561:AS-ALL +members: AS214953:AS-MRICKL +members: AS212234 +members: AS219406 +members: AS203019 +members: AS200625:AS-CONE +members: AS402902:AS-402902 +members: AS213286:AS-TRANSIT +members: AS216402:AS-LEWANPETIT +members: AS401023:AS-US +members: AS219488 +members: AS209632 +members: AS211585:AS-ALL +members: AS205163 +members: AS-ZHIWENSUN +members: AS203704:AS-SPANAGIOT +members: AS219503:AS-MAIN +members: AS219015 +members: AS34979 +members: AS214451:AS-PEERS +members: AS203374 +members: AS44822:AS-GXT +members: AS215146:AS-ROBERTOSANTO +members: AS17160:AS-NCOM +members: AS212993 +members: AS57330:AS-GIRAFFETECH +members: AS215786:AS-YOUCHEN +members: AS395878 +members: AS-LINETEX +members: AS215953:AS-ASHIMOV +members: AS215527:AS-XERONET +members: AS207149:AS-SNAWNET +members: AS219123:AS-MORNINETWORK +members: AS204994:AS-MAIN +members: AS402339:AS-ROUTE64 +members: AS-MSYS-IX +members: AS147020 +members: AS206098 +members: AS213434 +members: AS204150 +members: AS197137 +members: AS197291 +members: AS215502 +members: AS197305 +members: AS-FATIH-UNLU +members: AS-BRECHT +members: AS201033 +members: AS213959:AS-ALTSTACK +members: AS219249:AS-ALL +members: AS-MSRTG +members: AS199019 +members: AS212306:AS-SET +members: AS208563:AS-LINUXGEMINI +members: AS214630 +members: AS212170:AS-RULA +members: AS206290 +members: AS-AMIRGT +members: AS201423:AS-LUNALIGHT +members: AS209460:AS-ANNOUNCE +members: AS218830 +members: AS218816 +members: AS-AANETWORKS +members: AS200827 +members: AS-Y525 +members: AS212470:AS-LAPTOPCAT +members: AS402440 +members: AS-AKKGROUP +members: AS219458 +members: AS214342:AS-DIMINUENDO +members: AS209951 +members: AS203874:AS-DEVREX +members: AS197469 +members: AS203766 +members: AS213036 +members: AS202902 +members: AS219500 +members: AS208665 +members: AS214769 +members: AS211333 +members: AS215619 +members: AS219079 +members: AS207411 +members: AS216056 +members: AS210622:AS-KICKED +members: AS218718:AS-CONE +members: AS133504 +members: AS214382:AS-HOSTLAYICI +members: AS44001 +members: AS202118 +members: AS208471:AS-ALL +members: AS219442 +members: AS202458 +members: AS-FLORIANBAUER +members: AS206604 +members: AS207824:AS-DOWNSTREAM +members: AS201421:AS-CORE +members: AS-SET-IT7 +members: AS207529 +members: AS215899:AS-GUDIS +members: AS23026 +members: AS-PIEQ +members: AS199518:AS-199518 +members: AS214478:AS-SAKURACLOUDS +members: AS214533 +members: AS201132 +members: AS402182 +members: AS208904 +members: AS201439 +members: AS47341 +members: AS57908 +members: AS149771 +members: AS205280:AS-ALL +members: AS207272:AS-VOJTECH-LINHART +members: AS213422:AS-SOLO +members: AS-ZZNYJIDI +members: AS214380:AS-TIM427 +members: AS216190:AS-OCASDEV +members: AS213700 +members: AS275732 +members: AS213865 +members: AS207530:AS-TSERV-NET +members: AS201398 +members: AS402737 +members: AS203314 +members: AS215386:AS-AHOFMAN +members: AS204085:AS-ALL +members: AS200386:AS-T3HK0D3 +members: AS37707 +members: AS215844 +members: AS204856 +members: AS-FLORIAN +members: AS210360:AS-TRANSIT +members: AS207940:AS-KUPSCHKE +members: AS219239 +members: AS402790 +members: AS197324 +members: AS60077 +members: AS214161 +members: AS216005 +members: AS59449:AS-KONZOLA +members: AS218789:AS-ALL +members: AS218881 +members: AS201254 +members: AS402766 +members: AS-SYNAPSIS +members: AS207567:AS-ZETZOV +members: AS214387:AS-AKOL +members: AS215916:AS-JTM +members: AS207972 +members: AS200529:AS-INFOPRO +members: AS199395 +members: AS218778 +members: AS218901 +members: AS214731:AS-ETO +members: AS210199:AS-TRANSIT +members: AS152848 +members: AS200306 +members: AS210617:AS-PKERN +members: AS213447:AS-ASTER-PEERS +members: AS203738:AS-MEMBERS +members: AS-GREGORYF +members: AS52112 +members: AS198057 +members: AS212513 +members: AS219383 +members: AS-GLMN +members: AS198186:AS-SETESDALNET +members: AS-HLITSERVICES +members: AS401385 +members: AS219319 +members: AS203923 +members: AS199776 +members: AS211902 +members: AS204882:AS-DNOLAN +members: AS212888 +members: AS42301 +members: AS215296 +members: AS199109:AS-GUYNEWBURY +members: AS219094 +members: AS54281 +members: AS401575 +members: AS-CHIMON +members: AS402568:AS-AVERSON-IT +members: AS215499 +members: AS202473 +members: AS204033 +members: AS219367 +members: AS216342 +members: AS207549 +members: AS213586 +members: AS215760 +members: AS219133:AS-ALL +members: AS200992:AS-DWIM +members: AS213891:AS-LIUZHEN +members: AS199243:AS-GEEK-NZ +members: AS205369 +members: AS401404 +members: AS202222 +members: AS212510:AS-ADMINAFK +members: AS219505 +members: AS214938 +members: AS201445 +members: AS197658 +members: AS-VAMH +members: AS198657 +members: AS54380 +members: AS150141 +members: AS-WLG +members: AS211575:AS-RUINETWORK +members: AS214745 +members: AS212529:AS-ALL +members: AS210320:AS-FEXXIO +members: AS215887 +members: AS149998 +members: AS201694 +members: AS207884:AS-DASHPERF +members: AS54891 +members: AS12345 +members: AS44375 +members: AS201957:AS-ETHERNALCLOUD +members: AS215017 +members: AS-CHRISMUNDS +members: AS199241:AS-BF +members: AS207977 +members: AS219166 +members: AS25389 +members: AS214762 +members: AS211715 +members: AS214971 +members: AS211708 +members: AS202197 +members: AS60391 +members: AS212895 +members: AS210132 +members: AS-SNARK +members: AS219235:AS-DOWNSTREAM +members: AS212332:AS-THINCO +members: AS61333 +members: AS211411 +members: AS206208 +members: AS213449:AS-JOW +members: AS197127 +members: AS200728 +members: AS212386 +members: AS209057 +members: AS205079:AS-MANS +members: AS203226 +members: AS208824 +members: AS215769 +members: AS402069 +members: AS199195 +members: AS197676 +members: AS401315:AS-ILC-CONE +members: AS215509:AS-NRB +members: AS210715:AS-PISKOT +members: AS211939:AS-INZUGE +members: AS206569 +members: AS200063 +members: AS200464:AS-PAULO-R64 +members: AS215685:AS-DOTNET +members: AS219539 +members: AS200332 +members: AS214645:AS-TEST +members: AS34445 +members: AS200677 +members: AS208790:AS-DOWNSTREAMS +members: AS205874:AS-ZEROBYTE +remarks: **************************** +remarks: * THIS OBJECT IS MODIFIED +remarks: * Please note that all data that is generally regarded as personal +remarks: * data has been removed from this object. +remarks: * To view the original object, please query the RIPE Database at: +remarks: * http://www.ripe.net/whois +remarks: **************************** + +ADD 65934938 + +route: 155.117.178.0/24 +origin: AS402288 +mnt-by: netutils-mnt +created: 2026-09-14T22:04:22Z +last-modified: 2026-09-14T22:04:22Z +source: RIPE +remarks: **************************** +remarks: * THIS OBJECT IS MODIFIED +remarks: * Please note that all data that is generally regarded as personal +remarks: * data has been removed from this object. +remarks: * To view the original object, please query the RIPE Database at: +remarks: * http://www.ripe.net/whois +remarks: **************************** + +ADD 65934939 + +inetnum: 155.117.178.0 - 155.117.178.255 +netname: NET-155-117-178-0-24 +country: SG +geofeed: https://geofeed.ipxo.com/geofeed.txt +org: ORG-TTPL2-RIPE +admin-c: DUMY-RIPE +tech-c: DUMY-RIPE +abuse-c: TTPL7-RIPE +status: LEGACY +remarks: End User Organization +mnt-by: netutils-mnt +created: 2026-09-14T22:04:22Z +last-modified: 2026-09-14T22:04:22Z +source: RIPE +remarks: **************************** +remarks: * THIS OBJECT IS MODIFIED +remarks: * Please note that all data that is generally regarded as personal +remarks: * data has been removed from this object. +remarks: * To view the original object, please query the RIPE Database at: +remarks: * http://www.ripe.net/whois +remarks: **************************** + +ADD 65934940 + +organisation: ORG-MR209-RIPE +org-name: Makeev Rustam +org-type: OTHER +address: Dummy address for ORG-MR209-RIPE +e-mail: unread@ripe.net +mnt-ref: OVH-MNT +mnt-by: OVH-MNT +created: 2020-02-08T21:02:13Z +last-modified: 2026-09-14T22:04:27Z +source: RIPE +remarks: **************************** +remarks: * THIS OBJECT IS MODIFIED +remarks: * Please note that all data that is generally regarded as personal +remarks: * data has been removed from this object. +remarks: * To view the original object, please query the RIPE Database at: +remarks: * http://www.ripe.net/whois +remarks: **************************** + +DEL 65934941 + +organisation: ORG-MR209-RIPE +org-name: Makeev Rustam +org-type: OTHER +address: Dummy address for ORG-MR209-RIPE +e-mail: unread@ripe.net +mnt-ref: OVH-MNT +mnt-by: OVH-MNT +created: 2020-02-08T21:02:13Z +last-modified: 2026-09-14T22:04:27Z +source: RIPE +remarks: **************************** +remarks: * THIS OBJECT IS MODIFIED +remarks: * Please note that all data that is generally regarded as personal +remarks: * data has been removed from this object. +remarks: * To view the original object, please query the RIPE Database at: +remarks: * http://www.ripe.net/whois +remarks: **************************** + +ADD 65934942 + +route: 155.117.178.0/24 +origin: AS214483 +mnt-by: netutils-mnt +created: 2026-09-14T22:04:28Z +last-modified: 2026-09-14T22:04:28Z +source: RIPE +remarks: **************************** +remarks: * THIS OBJECT IS MODIFIED +remarks: * Please note that all data that is generally regarded as personal +remarks: * data has been removed from this object. +remarks: * To view the original object, please query the RIPE Database at: +remarks: * http://www.ripe.net/whois +remarks: **************************** + +ADD 65934944 + +mntner: mnt-sk-securcom1-1 +admin-c: DUMY-RIPE +upd-to: unread@ripe.net +auth: SSO noreply@ripe.net # Real value hidden for security +mnt-by: mnt-sk-securcom1-1 +created: 2019-03-07T09:05:33Z +last-modified: 2026-09-14T22:04:30Z +source: RIPE +remarks: **************************** +remarks: * THIS OBJECT IS MODIFIED +remarks: * Please note that all data that is generally regarded as personal +remarks: * data has been removed from this object. +remarks: * To view the original object, please query the RIPE Database at: +remarks: * http://www.ripe.net/whois +remarks: **************************** + +ADD 65934945 + +organisation: ORG-SS946-RIPE +org-name: securCom s.r.o. +country: SK +org-type: OTHER +address: Dummy address for ORG-SS946-RIPE +e-mail: unread@ripe.net +mnt-ref: mnt-sk-securcom1-1 +mnt-by: mnt-sk-securcom1-1 +created: 2019-03-07T09:05:34Z +last-modified: 2026-09-14T22:04:30Z +source: RIPE +remarks: **************************** +remarks: * THIS OBJECT IS MODIFIED +remarks: * Please note that all data that is generally regarded as personal +remarks: * data has been removed from this object. +remarks: * To view the original object, please query the RIPE Database at: +remarks: * http://www.ripe.net/whois +remarks: **************************** + +DEL 65934948 + +role: securCom a.s.,Abuse-C Role +address: Dummy address for AR51296-RIPE +org: ORG-SS946-RIPE +e-mail: unread@ripe.net +nic-hdl: AR51296-RIPE +abuse-mailbox: abuse@securcom.me +mnt-by: mnt-sk-securcom1-1 +created: 2019-03-07T09:05:33Z +last-modified: 2019-03-07T12:34:11Z +source: RIPE +remarks: **************************** +remarks: * THIS OBJECT IS MODIFIED +remarks: * Please note that all data that is generally regarded as personal +remarks: * data has been removed from this object. +remarks: * To view the original object, please query the RIPE Database at: +remarks: * http://www.ripe.net/whois +remarks: **************************** + +DEL 65934949 + +organisation: ORG-SS946-RIPE +org-name: securCom s.r.o. +country: SK +org-type: OTHER +address: Dummy address for ORG-SS946-RIPE +e-mail: unread@ripe.net +mnt-ref: mnt-sk-securcom1-1 +mnt-by: mnt-sk-securcom1-1 +created: 2019-03-07T09:05:34Z +last-modified: 2026-09-14T22:04:30Z +source: RIPE +remarks: **************************** +remarks: * THIS OBJECT IS MODIFIED +remarks: * Please note that all data that is generally regarded as personal +remarks: * data has been removed from this object. +remarks: * To view the original object, please query the RIPE Database at: +remarks: * http://www.ripe.net/whois +remarks: **************************** + +DEL 65934950 + +mntner: mnt-sk-securcom1-1 +admin-c: DUMY-RIPE +upd-to: unread@ripe.net +auth: SSO noreply@ripe.net # Real value hidden for security +mnt-by: mnt-sk-securcom1-1 +created: 2019-03-07T09:05:33Z +last-modified: 2026-09-14T22:04:30Z +source: RIPE +remarks: **************************** +remarks: * THIS OBJECT IS MODIFIED +remarks: * Please note that all data that is generally regarded as personal +remarks: * data has been removed from this object. +remarks: * To view the original object, please query the RIPE Database at: +remarks: * http://www.ripe.net/whois +remarks: **************************** + +ADD 65934953 + +aut-num: AS218772 +as-name: Barbonetworks +org: ORG-BB373-RIPE +sponsoring-org: ORG-NC22-RIPE +remarks: --------------------------------------- +remarks: Upstreams +remarks: --------------------------------------- +remarks: Netwerkvereniging Coloclue +mp-import: afi ipv4.unicast from AS8283 accept ANY +mp-export: afi ipv4.unicast to AS8283 announce AS218772:AS-BARBONETWORKS +mp-import: afi ipv6.unicast from AS8283 accept ANY +mp-export: afi ipv6.unicast to AS8283 announce AS218772:AS-BARBONETWORKS +remarks: Hurricane Electric +mp-import: afi ipv4.unicast from AS6939 accept ANY +mp-export: afi ipv4.unicast to AS6939 announce AS218772:AS-BARBONETWORKS +mp-import: afi ipv6.unicast from AS6939 accept ANY +mp-export: afi ipv6.unicast to AS6939 announce AS218772:AS-BARBONETWORKS +remarks: ROUTE64 +mp-import: afi ipv4.unicast from AS212895 accept ANY +mp-export: afi ipv4.unicast to AS212895 announce AS218772:AS-BARBONETWORKS +mp-import: afi ipv6.unicast from AS212895 accept ANY +mp-export: afi ipv6.unicast to AS212895 announce AS218772:AS-BARBONETWORKS +remarks: --------------------------------------- +remarks: IXP's +remarks: --------------------------------------- +remarks: BGP.EXCHANGE +mp-import: afi ipv6.unicast from AS24381 accept AS-BGPEXCHANGE +mp-export: afi ipv6.unicast to AS24381 announce AS218772:AS-BARBONETWORKS +remarks: 4b42 Internet Exchange +mp-import: afi ipv6.unicast from AS35708 accept AS-4IXP +mp-export: afi ipv6.unicast to AS35708 announce AS218772:AS-BARBONETWORKS +remarks: Frys-IX +mp-import: afi ipv6.unicast from AS56393 accept AS-FRYS-IX-CONNECTED +mp-export: afi ipv6.unicast to AS56393 announce AS218772:AS-BARBONETWORKS +remarks: --------------------------------------- +remarks: Peers +remarks: --------------------------------------- +remarks: LUJE.net +mp-import: afi ipv4.unicast from AS212855 accept AS212855:AS-LUJE +mp-export: afi ipv4.unicast to AS212855 announce AS218772:AS-BARBONETWORKS +mp-import: afi ipv6.unicast from AS212855 accept AS212855:AS-LUJE +mp-export: afi ipv6.unicast to AS212855 announce AS218772:AS-BARBONETWORKS +remarks: Hurricane Electric +mp-import: afi ipv4.unicast from AS6939 accept AS-HURRICANE +mp-export: afi ipv4.unicast to AS6939 announce AS218772:AS-BARBONETWORKS +mp-import: afi ipv6.unicast from AS6939 accept AS-HURRICANEv6 +mp-export: afi ipv6.unicast to AS6939 announce AS218772:AS-BARBONETWORKS +remarks: bgp-rodeo +mp-import: afi ipv4.unicast from AS202585 accept AS202585:AS-RODEO +mp-export: afi ipv4.unicast to AS202585 announce AS218772:AS-BARBONETWORKS +mp-import: afi ipv6.unicast from AS202585 accept AS202585:AS-RODEO +mp-export: afi ipv6.unicast to AS202585 announce AS218772:AS-BARBONETWORKS +remarks: Expeditioner Network +mp-import: afi ipv4.unicast from AS218836 accept AS218836:AS-EXPED +mp-export: afi ipv4.unicast to AS2218836 announce AS218772:AS-BARBONETWORKS +mp-import: afi ipv6.unicast from AS218836 accept AS218836:AS-EXPED +mp-export: afi ipv6.unicast to AS218836 announce AS218772:AS-BARBONETWORKS +remarks: Innovation Harbor B.V. +mp-import: afi ipv4.unicast from AS211588 accept AS211588:AS-XYPHEN +mp-export: afi ipv4.unicast to AS211588 announce AS218772:AS-BARBONETWORKS +mp-import: afi ipv6.unicast from AS211588 accept AS211588:AS-XYPHEN +mp-export: afi ipv6.unicast to AS211588 announce AS218772:AS-BARBONETWORKS +remarks: ROUTE64 +mp-import: afi ipv4.unicast from AS212895 accept AS212895:AS-ROUTE64 +mp-export: afi ipv4.unicast to AS212895 announce AS218772:AS-BARBONETWORKS +mp-import: afi ipv6.unicast from AS212895 accept AS212895:AS-ROUTE64 +mp-export: afi ipv6.unicast to AS212895 announce AS218772:AS-BARBONETWORKS +remarks: +admin-c: DUMY-RIPE +tech-c: DUMY-RIPE +status: ASSIGNED +mnt-by: RIPE-NCC-END-MNT +mnt-by: Barbo-MNT +created: 2026-09-03T07:24:46Z +last-modified: 2026-09-14T22:04:39Z +source: RIPE +remarks: **************************** +remarks: * THIS OBJECT IS MODIFIED +remarks: * Please note that all data that is generally regarded as personal +remarks: * data has been removed from this object. +remarks: * To view the original object, please query the RIPE Database at: +remarks: * http://www.ripe.net/whois +remarks: **************************** + +DEL 65934958 + +inetnum: 77.42.53.157 - 77.42.53.157 +netname: HOS-2192795 +descr: HOS-2192795 +country: DE +admin-c: DUMY-RIPE +tech-c: DUMY-RIPE +status: ASSIGNED PA +notify: ripe-mntner@hetzner.com +mnt-by: HOS-GUN +created: 2026-09-14T01:45:22Z +last-modified: 2026-09-14T01:45:22Z +source: RIPE +remarks: **************************** +remarks: * THIS OBJECT IS MODIFIED +remarks: * Please note that all data that is generally regarded as personal +remarks: * data has been removed from this object. +remarks: * To view the original object, please query the RIPE Database at: +remarks: * http://www.ripe.net/whois +remarks: **************************** + +DEL 65934959 + +inet6num: 2a01:4f8:242:4106::/64 +netname: FASTVPS-EESTI-OUE +descr: FastVPS Eesti OÜ +country: DE +admin-c: DUMY-RIPE +tech-c: DUMY-RIPE +status: ASSIGNED +notify: ripe-mntner@hetzner.com +mnt-by: HOS-GUN +created: 2023-04-12T01:07:02Z +last-modified: 2023-04-12T01:07:02Z +source: RIPE +remarks: **************************** +remarks: * THIS OBJECT IS MODIFIED +remarks: * Please note that all data that is generally regarded as personal +remarks: * data has been removed from this object. +remarks: * To view the original object, please query the RIPE Database at: +remarks: * http://www.ripe.net/whois +remarks: **************************** + +DEL 65934960 + +inet6num: 2a01:4f8:221:1fa2::/64 +netname: DIANA-HINOJOSA +descr: HOS-445612 +country: DE +admin-c: DUMY-RIPE +tech-c: DUMY-RIPE +status: ASSIGNED +notify: ripe-mntner@hetzner.com +mnt-by: HOS-GUN +created: 2026-07-12T01:15:06Z +last-modified: 2026-07-13T01:15:06Z +source: RIPE +remarks: **************************** +remarks: * THIS OBJECT IS MODIFIED +remarks: * Please note that all data that is generally regarded as personal +remarks: * data has been removed from this object. +remarks: * To view the original object, please query the RIPE Database at: +remarks: * http://www.ripe.net/whois +remarks: **************************** + From a934415cbe3f03b331aba969f1b9c900ed55d805 Mon Sep 17 00:00:00 2001 From: lou lecrivain Date: Thu, 17 Sep 2026 17:21:28 +0200 Subject: [PATCH 02/11] add deps needed for chunk pre-parsing and streaming feature --- Cargo.lock | 198 +++++++++++++++++++++++++++++++++++++++++++++++++---- Cargo.toml | 7 +- 2 files changed, 191 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 08b9478..ed3795f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,24 +2,101 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + [[package]] name = "bytes" version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "chardetng" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13de944a44b5064ee5d3a5ceccc49a41bfec50f2580e66f82e87703acdb88b53" +dependencies = [ + "cfg-if", + "encoding_rs", + "memchr", +] + +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + +[[package]] +name = "encoding_rs" +version = "0.8.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5ef0006ac9ab233c38522f5ae99cae3625151de8f706cacee1cba4b8e2832a" +dependencies = [ + "cfg-if", + "core_detect", + "multiversion", + "multiversion_no_op", + "rustversion", + "scopeguard", + "simdutf8", +] + [[package]] name = "futures-core" version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "futures-sink" version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-macro", + "futures-task", + "pin-project-lite", + "slab", +] + [[package]] name = "libc" version = "0.2.189" @@ -32,12 +109,44 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "multiversion" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ca4bea16ffc3f443cf7d866912118196bfef4c6a1556ca00f9f9b00bb43f7c" +dependencies = [ + "multiversion-macros", +] + +[[package]] +name = "multiversion-macros" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d416831a7317ef4b08bee00b69cbbb9c8763da7959a7026244d6266869f9c83" +dependencies = [ + "proc-macro2", + "quote", + "rustversion", + "syn 3.0.6", +] + +[[package]] +name = "multiversion_no_op" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" + [[package]] name = "nrtm-parser" version = "0.1.2" dependencies = [ + "chardetng", + "encoding_rs", + "futures-util", "pest", "pest_derive", + "re-delimiter-codec", + "regex", "tokio", "tokio-stream", "tokio-test", @@ -46,9 +155,9 @@ dependencies = [ [[package]] name = "pest" -version = "2.9.1" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d45aeb61b4bf818e12d4205f2466f8c4748f85f4fce0146d1c03d69d753f0ad" +checksum = "45d3aca230fad2e6f6317ca0a72724338c4960cb97168a85cdee66df4a9a21a8" dependencies = [ "memchr", "ucd-trie", @@ -56,9 +165,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.9.1" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89cc5a242e25ed4e7704d0be240f2cfbe20a8c27e7e252d94835be93d92dc39f" +checksum = "284b60557f2c4a2e72ad3f2d34d42685a2fa4a6a61d0d2a10c0ae2a5e916c2cf" dependencies = [ "pest", "pest_generator", @@ -66,9 +175,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.9.1" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7abf21475cc3820fe4b2ca2dc2142902f67a02189f3b5b3a229f4febc01a43e5" +checksum = "1d9d1f08a115309ee99268cf85e5228e0e56aa9caf8841ec12866b6be07c3109" dependencies = [ "pest", "pest_meta", @@ -79,9 +188,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.9.1" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adba4db388f687393c18c51348d44a41d870ca9df71a2c98172ea3035dc6936e" +checksum = "ed93ba1a9ffcca32130a5188701c81c0c49cf00d4b7c5007d5148951d743adcb" dependencies = [ "pest", ] @@ -110,6 +219,69 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "re-delimiter-codec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd265cf799a0cdc9eb1417b1403990fd29aafe6992877c70f2b75d1439125e77" +dependencies = [ + "regex", + "tokio-util", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "syn" version = "2.0.119" @@ -123,9 +295,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.5" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" dependencies = [ "proc-macro2", "quote", @@ -150,7 +322,7 @@ checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.6", ] [[package]] @@ -197,6 +369,6 @@ checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" [[package]] name = "unicode-ident" -version = "1.0.24" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" diff --git a/Cargo.toml b/Cargo.toml index bc84ec1..ee1d5bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,11 +15,16 @@ pest_derive = "2.9.0" tokio = { version = "1.53.1", optional = true } tokio-util = { version = "0.7.19", optional = true, features = ["codec"] } tokio-stream = { version = "0.1.19", optional = true } +re-delimiter-codec = { version = "0.1.0", optional = true } +regex = { version = "1.13.1", optional = true } +futures-util = { version = "0.3.34", optional = true } +chardetng = { version = "1.0.0", optional = true } +encoding_rs = { version = "0.8.41", features = ["std"], optional = true } [features] default = [ ] -async-streaming = [ "dep:tokio-util", "dep:tokio", "dep:tokio-stream" ] +async-streaming = [ "dep:tokio-util", "dep:tokio", "dep:tokio-stream", "dep:futures-util", "dep:re-delimiter-codec", "dep:regex", "dep:chardetng", "dep:encoding_rs" ] [dev-dependencies] tokio = { version = "1.53.1", features = ["fs", "macros", "rt" ] } From 10f9175c0fd5d1ec88d83ee7e49562659fe8035e Mon Sep 17 00:00:00 2001 From: lou lecrivain Date: Thu, 17 Sep 2026 18:26:57 +0200 Subject: [PATCH 03/11] implementation for chunk -> guesstimate -> parse --- src/lib.rs | 86 ++++++++++-- src/streaming.rs | 242 +++++++++++++++++--------------- src/tests.rs | 2 - src/tests/streaming_fixtures.rs | 10 -- tests/streaming_integration.rs | 51 ++++--- 5 files changed, 226 insertions(+), 165 deletions(-) delete mode 100644 src/tests/streaming_fixtures.rs diff --git a/src/lib.rs b/src/lib.rs index 4deaa1f..3edece9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,9 +7,13 @@ use pest_derive::Parser; use std::mem::discriminant; #[cfg(feature = "async-streaming")] -mod streaming; +pub mod streaming; #[cfg(feature = "async-streaming")] -use {tokio::io::AsyncRead, tokio_stream::StreamExt, tokio_util::codec::FramedRead}; +use { + crate::streaming::{NRTMDec, NRTMStreamError}, + futures_util::TryStream, + tokio::io::AsyncRead, +}; #[derive(Debug, Parser)] #[grammar = "./grammar.pest"] @@ -61,7 +65,6 @@ pub enum ParseError { MalformedSerial(Span, std::num::ParseIntError), LeadingGarbage(Span), IoError(std::io::Error), - NonUTF8Input(std::string::FromUtf8Error), } impl From for ParseError { fn from(io_err: std::io::Error) -> Self { @@ -78,37 +81,92 @@ pub trait StreamingNRTMParser where T: AsyncRead, { - fn reader_from(reader: T) -> impl StreamExt>; + fn stream_from( + &mut self, + reader: T, + ) -> impl TryStream; } -pub struct NRTMV3Parser; +pub struct NRTMV3Parser { + #[cfg(feature = "async-streaming")] + inner_decoder: NRTMDec, +} + +#[cfg(feature = "async-streaming")] +impl Default for NRTMV3Parser { + fn default() -> Self { + Self::new() + } +} + +impl NRTMV3Parser { + #[cfg(feature = "async-streaming")] + pub fn new() -> Self { + Self { + inner_decoder: NRTMDec::new_v3(), + } + } + #[cfg(not(feature = "async-streaming"))] + pub fn new() -> Self { + Self {} + } +} impl NRTMParser for NRTMV3Parser { fn try_parse(str: &str) -> Result { try_parse_nrtm(Rule::v3_operation, str) } } -#[cfg(feature = "async-streaming")] -impl StreamingNRTMParser for NRTMV2Parser { - fn reader_from(reader: T) -> impl StreamExt> { - FramedRead::new(reader, streaming::NRTMDec::new_v2()) - } +pub struct NRTMV2Parser { + #[cfg(feature = "async-streaming")] + inner_decoder: NRTMDec, } #[cfg(feature = "async-streaming")] -impl StreamingNRTMParser for NRTMV3Parser { - fn reader_from(reader: T) -> impl StreamExt> { - FramedRead::new(reader, streaming::NRTMDec::new_v3()) +impl Default for NRTMV2Parser { + fn default() -> Self { + Self::new() } } -pub struct NRTMV2Parser; +impl NRTMV2Parser { + #[cfg(feature = "async-streaming")] + pub fn new() -> Self { + Self { + inner_decoder: NRTMDec::new_v2(), + } + } + #[cfg(not(feature = "async-streaming"))] + pub fn new() -> Self { + Self {} + } +} impl NRTMParser for NRTMV2Parser { fn try_parse(str: &str) -> Result { try_parse_nrtm(Rule::v2_operation, str) } } +#[cfg(feature = "async-streaming")] +impl StreamingNRTMParser for NRTMV2Parser { + fn stream_from( + &mut self, + reader: T, + ) -> impl TryStream { + self.inner_decoder.get_stream(reader) + } +} + +#[cfg(feature = "async-streaming")] +impl StreamingNRTMParser for NRTMV3Parser { + fn stream_from( + &mut self, + reader: T, + ) -> impl TryStream { + self.inner_decoder.get_stream(reader) + } +} + fn try_parse_nrtm(root_type: Rule, str: &str) -> Result { use pest::Parser; diff --git a/src/streaming.rs b/src/streaming.rs index 4e80562..e5c7921 100644 --- a/src/streaming.rs +++ b/src/streaming.rs @@ -1,174 +1,174 @@ use crate::{NRTMMessage, NRTMParser, NRTMV2Parser, NRTMV3Parser, ParseError}; -use tokio_util::bytes::BytesMut; -use tokio_util::codec::Decoder; - -const MIN_BUFFER_LEN: usize = 8192; -const MIN_DECODE_LEN: usize = "ADD 1".len(); +use chardetng::{EncodingDetector, Iso2022JpDetection, Utf8Detection}; +use futures_util::{TryStream, TryStreamExt}; +use re_delimiter_codec::{REDelimiterCodec, REDelimiterCodecError}; +use regex::bytes::Regex; +use tokio::io::AsyncRead; +use tokio_util::codec::FramedRead; #[derive(Clone)] -pub struct NRTMDec { +pub(crate) struct NRTMDec { parser: fn(&str) -> Result, } +#[derive(Debug)] +pub enum NRTMStreamError { + REDelimiterCodec(REDelimiterCodecError), + Parser(ParseError), +} + +fn new_nrtm_preparser() -> REDelimiterCodec { + const MAX_CHUNK_LEN: usize = 131072; // 128k + + // ok to call unwrap here, we know this will not fail + REDelimiterCodec::new_with_max_length( + Regex::new("(?R)\n[^%][^AD][^DE][^DL].*\n\n").unwrap(), + MAX_CHUNK_LEN, + ) +} + impl NRTMDec { - pub fn new_v2() -> Self { + pub(crate) fn new_v2() -> Self { NRTMDec { parser: NRTMV2Parser::try_parse, } } - pub fn new_v3() -> Self { + pub(crate) fn new_v3() -> Self { NRTMDec { parser: NRTMV3Parser::try_parse, } } -} - -impl Decoder for NRTMDec { - type Item = NRTMMessage; - type Error = ParseError; - - fn decode(&mut self, src: &mut BytesMut) -> Result, Self::Error> { - src.reserve(MIN_BUFFER_LEN); - - // from_utf8 is no-copy - let str = String::from_utf8(src.to_vec()).map_err(ParseError::NonUTF8Input)?; - - // bail early if buffer is empty, as a parser run on empty and / or undecidable str - // is a waste of cycles and may yield errors - if src.is_empty() || src.len() <= MIN_DECODE_LEN { - return Ok(None); - } - - match (self.parser)(str.as_str()) { - Ok(message) => { - let _message_bytes = src.split_to(message.span.end_b); - // implicit drop for message_str and message_bytes - Ok(Some(message)) - } - // per tokio-util codec documentation, - // If the bytes look valid, but a frame isn’t fully available yet, - // then Ok(None) is returned. - Err(ParseError::Incomplete) => Ok(None), - Err(ParseError::LeadingGarbage(span)) => { - // split garbage, flush and return err - let _garbage_bytes = src.split_to(span.end_b); - Err(ParseError::LeadingGarbage(span)) - } - // malformed input, split at malformed, flush and return err - // the rest will be eaten as leading garbage - Err(ParseError::Parser(err)) => { - let _malformed_bytes = match err.location { - // boop it! kick it! drop it! - pest::error::InputLocation::Pos(u) => src.split_to(u), - pest::error::InputLocation::Span((_, end)) => src.split_to(end), + pub(crate) fn get_stream( + &mut self, + reader: T, + ) -> impl TryStream { + let framed_reader = FramedRead::new(reader, new_nrtm_preparser()); + let parser = self.parser; + + framed_reader + .and_then( + // charset guesstimation + |chunk| { + // for each chunk we need a new instance of detector, + // as each chunk potentially has a different charset + let mut encoding_detector = EncodingDetector::new(Iso2022JpDetection::Allow); + // re ISO 2022 JP, we don't care about preventing XSS, this is the job of either + // the data source or the client. we're only a middle layer and so + // should not meddle with the data. + + // feed all the chunk data in the encoding detector + encoding_detector.feed(chunk.as_ref(), true); + + // there is no encoding spec for NRTMv2 and NRTMv3, so allowing UTF-8 as a + // detection target is fine + let encoding = encoding_detector.guess(None, Utf8Detection::Allow); + + // result is String, in most cases no copy will happen. copy will only happen + // for replacement characters insertion + let (decoded, _, _) = encoding.decode(chunk.as_ref()); + + futures_util::future::ready(Ok(decoded.into_owned())) + }, + ) + .map_err(NRTMStreamError::REDelimiterCodec) // use same error type for encapsulation + .and_then(move |cow_str| { + let r = match parser(cow_str.as_ref()) { + Ok(message) => Ok(message), + Err(e) => Err(NRTMStreamError::Parser(e)), // client should recover }; - Err(ParseError::Parser(err)) - } - // malformed utf-8, flush incriminated section - Err(ParseError::NonUTF8Input(u8)) => { - let _utf8err_bytes = &mut src.split_to(u8.utf8_error().valid_up_to()); - Err(ParseError::NonUTF8Input(u8)) - } - // malformed serial - Err(ParseError::MalformedSerial(span, e)) => { - let _garbage_bytes = src.split_to(span.end_b); // ibid - Err(ParseError::MalformedSerial(span, e)) - } - Err(ParseError::NoMatch) => Err(ParseError::NoMatch), - Err(ParseError::IoError(ioe)) => { - src.clear(); // clear buffer - Err(ParseError::IoError(ioe)) - } - } + futures_util::future::ready(r) + }) } } #[cfg(test)] mod tests { - use crate::streaming::NRTMDec; - use crate::tests::streaming_fixtures; - use crate::{OpType, ParseError, Verb}; + use super::*; + use crate::{NRTMMessage, OpType, ParseError}; + use futures_util::TryStreamExt; use std::assert_matches; use std::io::{Error as IOError, ErrorKind}; use tokio::fs::File; - use tokio_stream::StreamExt; use tokio_test::io::Builder; - use tokio_util::codec::FramedRead; #[tokio::test] - async fn v3_message_stream_ok() { - let nrtmv3_sample = File::open("./src/tests/nrtmv3_ripe_sample.txt") + async fn v3_charset_guesstimation_ok() { + let nrtmv3_sample = File::open("./src/tests/nrtmv3_ripe_mixed_encoding_sample.txt") .await .unwrap(); - let decoder = NRTMDec::new_v3(); - let mut reader = FramedRead::new(nrtmv3_sample, decoder); - let mut linear_id_counter = 65776763; - - while let Some(Ok(res)) = reader.next().await { - match res.update { - OpType::V3(Verb::ADD, ctr) => { - linear_id_counter += 1; - assert_eq!(ctr, linear_id_counter); + let mut decoder = NRTMDec::new_v3(); + let mut stream = decoder.get_stream(nrtmv3_sample); + let mut linear_increase_id_counter = 65934900; + + loop { + match stream.try_next().await { + Ok(Some(NRTMMessage { + update: OpType::V3(_, ctr), + rpsl: _, + span: _, + })) => { + assert!(linear_increase_id_counter < ctr); // check strictly increasing + linear_increase_id_counter = ctr; } - _ => panic!("incorrect update {:?}", res.update), + Ok(Some(NRTMMessage { + update: OpType::V2(_), + rpsl: _, + span: _, + })) => {} // ignore + // comment at the beginning is marked as a chunk but is incomplete from the POV + // of the parser + Err(e) => println!("got error {:?}", e), + Ok(None) => break, // end of stream } } - assert_eq!(linear_id_counter, 65776785); // last object id + + assert_eq!(linear_increase_id_counter, 65934960); // last object id } #[tokio::test] async fn v3_parser_error_signalled() { - let mut reader = streaming_fixtures::v3_reader_from( - b"\ + let mut decoder = NRTMDec::new_v3(); + let mut reader = decoder.get_stream( + &b"\ ADD 324876 object: property \\xxt*some-more: properties end-of: object -", +"[..], ); - assert_matches!(reader.next().await, Some(Err(ParseError::Parser(_)))); - } - - #[tokio::test] - async fn v3_non_utf8_error_signalled() { - let mut reader = streaming_fixtures::v3_reader_from( - b"\ -ADD 324876 - -object: property\xF8\xF8 -some-more: properties -end-of: object - -", + assert_matches!( + reader.try_next().await, + Err(NRTMStreamError::Parser(ParseError::Parser(_))) ); - assert_matches!(reader.next().await, Some(Err(ParseError::NonUTF8Input(_)))); } #[tokio::test] async fn v3_malformed_serial_signalled() { - let mut reader = streaming_fixtures::v3_reader_from( - b"\ + let mut decoder = NRTMDec::new_v3(); + let mut reader = decoder.get_stream( + &b"\ # should not fit into u64 ADD 99999999999999999999 start-field: yes netname: TRANSPORT-NET -", +"[..], ); assert_matches!( - reader.next().await, - Some(Err(ParseError::MalformedSerial(_, _))) + reader.try_next().await, + Err(NRTMStreamError::Parser(ParseError::MalformedSerial(_, _))) ); } #[tokio::test] - async fn v3_no_match_signalled() { - let mut reader = streaming_fixtures::v3_reader_from( - b"\ + async fn v3_no_chunks_signalled() { + let mut decoder = NRTMDec::new_v3(); + let mut reader = decoder.get_stream( + &b"\ % The RIPE Database is subject to Terms and Conditions. % See https://docs.db.ripe.net/terms-conditions.html @@ -177,10 +177,15 @@ netname: TRANSPORT-NET % -kg RIPE:3:65776764-LAST %START Version: 3 RIPE 65776764-65776784 # comment -", +"[..], ); - assert_matches!(reader.next().await, Some(Err(ParseError::IoError(_)))); + assert_matches!( + reader.try_next().await, + Err(NRTMStreamError::REDelimiterCodec( + REDelimiterCodecError::Io(_) + )) + ); } #[tokio::test] @@ -202,10 +207,15 @@ end-of: obj .read(nrtmv3_truncated_message) .read_error(IOError::new(ErrorKind::BrokenPipe, "connection closed")) .build(); - let decoder = NRTMDec::new_v3(); - let mut reader = FramedRead::new(ioerroring_sample, decoder); + let mut decoder = NRTMDec::new_v3(); + let mut reader = decoder.get_stream(ioerroring_sample); - reader.next().await.unwrap().unwrap(); // chuck first object - assert_matches!(reader.next().await, Some(Err(ParseError::IoError(_)))); + reader.try_next().await.unwrap().unwrap(); // chuck first object + assert_matches!( + reader.try_next().await, + Err(NRTMStreamError::REDelimiterCodec( + REDelimiterCodecError::Io(_) + )) + ); } } diff --git a/src/tests.rs b/src/tests.rs index d15e22b..5118a6e 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -1,4 +1,2 @@ mod parse; mod rules; -#[cfg(feature = "async-streaming")] -pub(crate) mod streaming_fixtures; diff --git a/src/tests/streaming_fixtures.rs b/src/tests/streaming_fixtures.rs deleted file mode 100644 index 7b85121..0000000 --- a/src/tests/streaming_fixtures.rs +++ /dev/null @@ -1,10 +0,0 @@ -use crate::streaming::NRTMDec; -use crate::{NRTMMessage, ParseError}; -use tokio_stream::StreamExt; -use tokio_util::codec::FramedRead; - -pub fn v3_reader_from( - bytes_slice: &[u8], -) -> impl StreamExt> { - FramedRead::new(bytes_slice, NRTMDec::new_v3()) -} diff --git a/tests/streaming_integration.rs b/tests/streaming_integration.rs index a5821b1..a209b00 100644 --- a/tests/streaming_integration.rs +++ b/tests/streaming_integration.rs @@ -1,47 +1,52 @@ #[cfg(feature = "async-streaming")] use { - nrtm_parser::StreamingNRTMParser, - nrtm_parser::{NRTMV3Parser, OpType, Verb}, + futures_util::TryStreamExt, + nrtm_parser::streaming::NRTMStreamError, + nrtm_parser::{NRTMV3Parser, OpType, ParseError, StreamingNRTMParser, Verb}, tokio::fs::File, - tokio_stream::StreamExt, }; #[cfg(feature = "async-streaming")] #[tokio::test] async fn parse_message_stream_example() { - let nrtmv3_sample = File::open("./src/tests/nrtmv3_ripe_sample.txt") + let nrtmv3_sample = File::open("./src/tests/nrtmv3_ripe_mixed_encoding_sample.txt") .await .unwrap(); - let mut parser = NRTMV3Parser::reader_from(nrtmv3_sample); + let mut parser = NRTMV3Parser::new(); + let mut stream = parser.stream_from(nrtmv3_sample); loop { - let optional_result = parser.next().await; + let optional_result = stream.try_next().await; match optional_result { - None => break, // end of stream - Some(result) => match result { - Ok(nrtm_message) => match nrtm_message.update { + Err(NRTMStreamError::Parser(ParseError::NoMatch)) + | Err(NRTMStreamError::Parser(ParseError::Incomplete)) + | Err(NRTMStreamError::Parser(ParseError::Parser(_))) + | Err(NRTMStreamError::Parser(ParseError::MalformedSerial(_, _))) + | Err(NRTMStreamError::Parser(ParseError::LeadingGarbage(_))) => { + // do nothing, all of these are retryable errors + // (we should have at least one incomplete parse error due to prelude / comment + // being chunked). ideally for each error type you'd want to log it, so that + // we know some garbage data was in the stream. + } + Err(error) => panic!( + "irrecoverable error during consumption of NRTM stream {:?}", + error + ), + Ok(None) => break, // end of stream + Ok(Some(message)) => { + match message.update { OpType::V2(_) => {} // ignore v2 OpType::V3(verb, serial) => match verb { Verb::ADD => { - println!( - "operation {serial}, adding rpsl object {}", - nrtm_message.rpsl - ); + println!("operation {serial}, adding rpsl object {}", message.rpsl); } Verb::DEL => { - println!( - "operation {serial}, deleting rpsl object {}", - nrtm_message.rpsl - ); + println!("operation {serial}, deleting rpsl object {}", message.rpsl); } }, - }, - Err(error) => panic!( - "should have parsed successfully but received error {:?}", - error - ), - }, + } + } } } } From 3005c7018399522f1041e7aab8ebff56cf08f3b4 Mon Sep 17 00:00:00 2001 From: lou lecrivain Date: Tue, 22 Sep 2026 11:57:26 +0200 Subject: [PATCH 04/11] delete no longer used test file --- src/tests/nrtmv3_ripe_sample.txt | 577 ------------------------------- 1 file changed, 577 deletions(-) delete mode 100644 src/tests/nrtmv3_ripe_sample.txt diff --git a/src/tests/nrtmv3_ripe_sample.txt b/src/tests/nrtmv3_ripe_sample.txt deleted file mode 100644 index 12951ba..0000000 --- a/src/tests/nrtmv3_ripe_sample.txt +++ /dev/null @@ -1,577 +0,0 @@ -% The RIPE Database is subject to Terms and Conditions. -% See https://docs.db.ripe.net/terms-conditions.html - -% lou: I am commenting that out, as in rx only stream -% commands will not appear -% -kg RIPE:3:65776764-LAST -%START Version: 3 RIPE 65776764-65776784 - -ADD 65776764 - -inetnum: 213.178.11.0 - 213.178.11.255 -netname: PSVNEO-CUSTOMER-NET-BACKUP -country: DE -admin-c: DUMY-RIPE -tech-c: DUMY-RIPE -status: ASSIGNED PA -created: 2026-08-19T14:41:10Z -last-modified: 2026-08-19T14:41:10Z -source: RIPE -mnt-by: IMSC-MNT -remarks: **************************** -remarks: * THIS OBJECT IS MODIFIED -remarks: * Please note that all data that is generally regarded as personal -remarks: * data has been removed from this object. -remarks: * To view the original object, please query the RIPE Database at: -remarks: * http://www.ripe.net/whois -remarks: **************************** - -ADD 65776765 - -inet6num: 2a0f:6280:c814::/48 -netname: FUTURE-USAGE -descr: Iridium -country: TR -admin-c: DUMY-RIPE -tech-c: DUMY-RIPE -abuse-c: ACRO56593-RIPE -status: ASSIGNED -mnt-by: HYEHOST-MNT -org: ORG-IA2236-RIPE -created: 2026-08-19T14:41:30Z -last-modified: 2026-08-19T14:41:30Z -source: RIPE -geofeed: https://api.geofeed.space/fc79d4c7-ff20-4ed7-bc0a-1c44b19d377c/geofeed.csv -remarks: **************************** -remarks: * THIS OBJECT IS MODIFIED -remarks: * Please note that all data that is generally regarded as personal -remarks: * data has been removed from this object. -remarks: * To view the original object, please query the RIPE Database at: -remarks: * http://www.ripe.net/whois -remarks: **************************** - -ADD 65776766 - -inet6num: 2a0f:6280:c815::/48 -netname: FUTURE-USAGE -descr: Iridium -country: DK -admin-c: DUMY-RIPE -tech-c: DUMY-RIPE -abuse-c: ACRO56593-RIPE -status: ASSIGNED -mnt-by: HYEHOST-MNT -org: ORG-IA2236-RIPE -created: 2026-08-19T14:41:30Z -last-modified: 2026-08-19T14:41:30Z -source: RIPE -geofeed: https://api.geofeed.space/fc79d4c7-ff20-4ed7-bc0a-1c44b19d377c/geofeed.csv -remarks: **************************** -remarks: * THIS OBJECT IS MODIFIED -remarks: * Please note that all data that is generally regarded as personal -remarks: * data has been removed from this object. -remarks: * To view the original object, please query the RIPE Database at: -remarks: * http://www.ripe.net/whois -remarks: **************************** - -ADD 65776767 - -inet6num: 2a0f:6280:c816::/48 -netname: FUTURE-USAGE -descr: Iridium -country: VG -admin-c: DUMY-RIPE -tech-c: DUMY-RIPE -abuse-c: ACRO56593-RIPE -status: ASSIGNED -mnt-by: HYEHOST-MNT -org: ORG-IA2236-RIPE -created: 2026-08-19T14:41:30Z -last-modified: 2026-08-19T14:41:30Z -source: RIPE -geofeed: https://api.geofeed.space/fc79d4c7-ff20-4ed7-bc0a-1c44b19d377c/geofeed.csv -remarks: **************************** -remarks: * THIS OBJECT IS MODIFIED -remarks: * Please note that all data that is generally regarded as personal -remarks: * data has been removed from this object. -remarks: * To view the original object, please query the RIPE Database at: -remarks: * http://www.ripe.net/whois -remarks: **************************** - -ADD 65776768 - -inet6num: 2a0f:6280:c817::/48 -netname: FUTURE-USAGE -descr: Iridium -country: SB -admin-c: DUMY-RIPE -tech-c: DUMY-RIPE -abuse-c: ACRO56593-RIPE -status: ASSIGNED -mnt-by: HYEHOST-MNT -org: ORG-IA2236-RIPE -created: 2026-08-19T14:41:31Z -last-modified: 2026-08-19T14:41:31Z -source: RIPE -geofeed: https://api.geofeed.space/fc79d4c7-ff20-4ed7-bc0a-1c44b19d377c/geofeed.csv -remarks: **************************** -remarks: * THIS OBJECT IS MODIFIED -remarks: * Please note that all data that is generally regarded as personal -remarks: * data has been removed from this object. -remarks: * To view the original object, please query the RIPE Database at: -remarks: * http://www.ripe.net/whois -remarks: **************************** - -ADD 65776769 - -inet6num: 2a0f:6280:c818::/48 -netname: FUTURE-USAGE -descr: Iridium -country: EG -admin-c: DUMY-RIPE -tech-c: DUMY-RIPE -abuse-c: ACRO56593-RIPE -status: ASSIGNED -mnt-by: HYEHOST-MNT -org: ORG-IA2236-RIPE -created: 2026-08-19T14:41:31Z -last-modified: 2026-08-19T14:41:31Z -source: RIPE -geofeed: https://api.geofeed.space/fc79d4c7-ff20-4ed7-bc0a-1c44b19d377c/geofeed.csv -remarks: **************************** -remarks: * THIS OBJECT IS MODIFIED -remarks: * Please note that all data that is generally regarded as personal -remarks: * data has been removed from this object. -remarks: * To view the original object, please query the RIPE Database at: -remarks: * http://www.ripe.net/whois -remarks: **************************** - -ADD 65776770 - -inet6num: 2a0f:6280:c819::/48 -netname: FUTURE-USAGE -descr: Iridium -country: FM -admin-c: DUMY-RIPE -tech-c: DUMY-RIPE -abuse-c: ACRO56593-RIPE -status: ASSIGNED -mnt-by: HYEHOST-MNT -org: ORG-IA2236-RIPE -created: 2026-08-19T14:41:31Z -last-modified: 2026-08-19T14:41:31Z -source: RIPE -geofeed: https://api.geofeed.space/fc79d4c7-ff20-4ed7-bc0a-1c44b19d377c/geofeed.csv -remarks: **************************** -remarks: * THIS OBJECT IS MODIFIED -remarks: * Please note that all data that is generally regarded as personal -remarks: * data has been removed from this object. -remarks: * To view the original object, please query the RIPE Database at: -remarks: * http://www.ripe.net/whois -remarks: **************************** - -ADD 65776771 - -inet6num: 2a0f:6280:c81a::/48 -netname: FUTURE-USAGE -descr: Iridium -country: FK -admin-c: DUMY-RIPE -tech-c: DUMY-RIPE -abuse-c: ACRO56593-RIPE -status: ASSIGNED -mnt-by: HYEHOST-MNT -org: ORG-IA2236-RIPE -created: 2026-08-19T14:41:32Z -last-modified: 2026-08-19T14:41:32Z -source: RIPE -geofeed: https://api.geofeed.space/fc79d4c7-ff20-4ed7-bc0a-1c44b19d377c/geofeed.csv -remarks: **************************** -remarks: * THIS OBJECT IS MODIFIED -remarks: * Please note that all data that is generally regarded as personal -remarks: * data has been removed from this object. -remarks: * To view the original object, please query the RIPE Database at: -remarks: * http://www.ripe.net/whois -remarks: **************************** - -ADD 65776772 - -inet6num: 2a0f:6280:c81b::/48 -netname: FUTURE-USAGE -descr: Iridium -country: ZA -admin-c: DUMY-RIPE -tech-c: DUMY-RIPE -abuse-c: ACRO56593-RIPE -status: ASSIGNED -mnt-by: HYEHOST-MNT -org: ORG-IA2236-RIPE -created: 2026-08-19T14:41:32Z -last-modified: 2026-08-19T14:41:32Z -source: RIPE -geofeed: https://api.geofeed.space/fc79d4c7-ff20-4ed7-bc0a-1c44b19d377c/geofeed.csv -remarks: **************************** -remarks: * THIS OBJECT IS MODIFIED -remarks: * Please note that all data that is generally regarded as personal -remarks: * data has been removed from this object. -remarks: * To view the original object, please query the RIPE Database at: -remarks: * http://www.ripe.net/whois -remarks: **************************** - -ADD 65776773 - -inet6num: 2a0f:6280:c81c::/48 -netname: FUTURE-USAGE -descr: Iridium -country: TZ -admin-c: DUMY-RIPE -tech-c: DUMY-RIPE -abuse-c: ACRO56593-RIPE -status: ASSIGNED -mnt-by: HYEHOST-MNT -org: ORG-IA2236-RIPE -created: 2026-08-19T14:41:32Z -last-modified: 2026-08-19T14:41:32Z -source: RIPE -geofeed: https://api.geofeed.space/fc79d4c7-ff20-4ed7-bc0a-1c44b19d377c/geofeed.csv -remarks: **************************** -remarks: * THIS OBJECT IS MODIFIED -remarks: * Please note that all data that is generally regarded as personal -remarks: * data has been removed from this object. -remarks: * To view the original object, please query the RIPE Database at: -remarks: * http://www.ripe.net/whois -remarks: **************************** - -ADD 65776774 - -inet6num: 2a0f:6280:c81d::/48 -netname: FUTURE-USAGE -descr: Iridium -country: TW -admin-c: DUMY-RIPE -tech-c: DUMY-RIPE -abuse-c: ACRO56593-RIPE -status: ASSIGNED -mnt-by: HYEHOST-MNT -org: ORG-IA2236-RIPE -created: 2026-08-19T14:41:32Z -last-modified: 2026-08-19T14:41:32Z -source: RIPE -geofeed: https://api.geofeed.space/fc79d4c7-ff20-4ed7-bc0a-1c44b19d377c/geofeed.csv -remarks: **************************** -remarks: * THIS OBJECT IS MODIFIED -remarks: * Please note that all data that is generally regarded as personal -remarks: * data has been removed from this object. -remarks: * To view the original object, please query the RIPE Database at: -remarks: * http://www.ripe.net/whois -remarks: **************************** - -ADD 65776775 - -inet6num: 2a0f:6280:c81e::/48 -netname: FUTURE-USAGE -descr: Iridium -country: RO -admin-c: DUMY-RIPE -tech-c: DUMY-RIPE -abuse-c: ACRO56593-RIPE -status: ASSIGNED -mnt-by: HYEHOST-MNT -org: ORG-IA2236-RIPE -created: 2026-08-19T14:41:33Z -last-modified: 2026-08-19T14:41:33Z -source: RIPE -geofeed: https://api.geofeed.space/fc79d4c7-ff20-4ed7-bc0a-1c44b19d377c/geofeed.csv -remarks: **************************** -remarks: * THIS OBJECT IS MODIFIED -remarks: * Please note that all data that is generally regarded as personal -remarks: * data has been removed from this object. -remarks: * To view the original object, please query the RIPE Database at: -remarks: * http://www.ripe.net/whois -remarks: **************************** - -ADD 65776776 - -inet6num: 2a0f:6280:c81f::/48 -netname: FUTURE-USAGE -descr: Iridium -country: NA -admin-c: DUMY-RIPE -tech-c: DUMY-RIPE -abuse-c: ACRO56593-RIPE -status: ASSIGNED -mnt-by: HYEHOST-MNT -org: ORG-IA2236-RIPE -created: 2026-08-19T14:41:33Z -last-modified: 2026-08-19T14:41:33Z -source: RIPE -geofeed: https://api.geofeed.space/fc79d4c7-ff20-4ed7-bc0a-1c44b19d377c/geofeed.csv -remarks: **************************** -remarks: * THIS OBJECT IS MODIFIED -remarks: * Please note that all data that is generally regarded as personal -remarks: * data has been removed from this object. -remarks: * To view the original object, please query the RIPE Database at: -remarks: * http://www.ripe.net/whois -remarks: **************************** - -ADD 65776777 - -inet6num: 2a0f:6280:c820::/48 -netname: FUTURE-USAGE -descr: Iridium -country: PH -admin-c: DUMY-RIPE -tech-c: DUMY-RIPE -abuse-c: ACRO56593-RIPE -status: ASSIGNED -mnt-by: HYEHOST-MNT -org: ORG-IA2236-RIPE -created: 2026-08-19T14:41:33Z -last-modified: 2026-08-19T14:41:33Z -source: RIPE -geofeed: https://api.geofeed.space/fc79d4c7-ff20-4ed7-bc0a-1c44b19d377c/geofeed.csv -remarks: **************************** -remarks: * THIS OBJECT IS MODIFIED -remarks: * Please note that all data that is generally regarded as personal -remarks: * data has been removed from this object. -remarks: * To view the original object, please query the RIPE Database at: -remarks: * http://www.ripe.net/whois -remarks: **************************** - -ADD 65776778 - -inet6num: 2a0f:6280:c821::/48 -netname: FUTURE-USAGE -descr: Iridium -country: BW -admin-c: DUMY-RIPE -tech-c: DUMY-RIPE -abuse-c: ACRO56593-RIPE -status: ASSIGNED -mnt-by: HYEHOST-MNT -org: ORG-IA2236-RIPE -created: 2026-08-19T14:41:33Z -last-modified: 2026-08-19T14:41:33Z -source: RIPE -geofeed: https://api.geofeed.space/fc79d4c7-ff20-4ed7-bc0a-1c44b19d377c/geofeed.csv -remarks: **************************** -remarks: * THIS OBJECT IS MODIFIED -remarks: * Please note that all data that is generally regarded as personal -remarks: * data has been removed from this object. -remarks: * To view the original object, please query the RIPE Database at: -remarks: * http://www.ripe.net/whois -remarks: **************************** - -ADD 65776779 - -inet6num: 2a0f:6280:c822::/48 -netname: FUTURE-USAGE -descr: Iridium -country: RU -admin-c: DUMY-RIPE -tech-c: DUMY-RIPE -abuse-c: ACRO56593-RIPE -status: ASSIGNED -mnt-by: HYEHOST-MNT -org: ORG-IA2236-RIPE -created: 2026-08-19T14:41:34Z -last-modified: 2026-08-19T14:41:34Z -source: RIPE -geofeed: https://api.geofeed.space/fc79d4c7-ff20-4ed7-bc0a-1c44b19d377c/geofeed.csv -remarks: **************************** -remarks: * THIS OBJECT IS MODIFIED -remarks: * Please note that all data that is generally regarded as personal -remarks: * data has been removed from this object. -remarks: * To view the original object, please query the RIPE Database at: -remarks: * http://www.ripe.net/whois -remarks: **************************** - -ADD 65776780 - -inet6num: 2a0f:6280:c823::/48 -netname: FUTURE-USAGE -descr: Iridium -country: GA -admin-c: DUMY-RIPE -tech-c: DUMY-RIPE -abuse-c: ACRO56593-RIPE -status: ASSIGNED -mnt-by: HYEHOST-MNT -org: ORG-IA2236-RIPE -created: 2026-08-19T14:41:34Z -last-modified: 2026-08-19T14:41:34Z -source: RIPE -geofeed: https://api.geofeed.space/fc79d4c7-ff20-4ed7-bc0a-1c44b19d377c/geofeed.csv -remarks: **************************** -remarks: * THIS OBJECT IS MODIFIED -remarks: * Please note that all data that is generally regarded as personal -remarks: * data has been removed from this object. -remarks: * To view the original object, please query the RIPE Database at: -remarks: * http://www.ripe.net/whois -remarks: **************************** - -ADD 65776781 - -inet6num: 2a0f:6280:c824::/48 -netname: FUTURE-USAGE -descr: Iridium -country: GH -admin-c: DUMY-RIPE -tech-c: DUMY-RIPE -abuse-c: ACRO56593-RIPE -status: ASSIGNED -mnt-by: HYEHOST-MNT -org: ORG-IA2236-RIPE -created: 2026-08-19T14:41:34Z -last-modified: 2026-08-19T14:41:34Z -source: RIPE -geofeed: https://api.geofeed.space/fc79d4c7-ff20-4ed7-bc0a-1c44b19d377c/geofeed.csv -remarks: **************************** -remarks: * THIS OBJECT IS MODIFIED -remarks: * Please note that all data that is generally regarded as personal -remarks: * data has been removed from this object. -remarks: * To view the original object, please query the RIPE Database at: -remarks: * http://www.ripe.net/whois -remarks: **************************** - -ADD 65776782 - -inet6num: 2a0f:6280:c825::/48 -netname: FUTURE-USAGE -descr: Iridium -country: KH -admin-c: DUMY-RIPE -tech-c: DUMY-RIPE -abuse-c: ACRO56593-RIPE -status: ASSIGNED -mnt-by: HYEHOST-MNT -org: ORG-IA2236-RIPE -created: 2026-08-19T14:41:34Z -last-modified: 2026-08-19T14:41:34Z -source: RIPE -geofeed: https://api.geofeed.space/fc79d4c7-ff20-4ed7-bc0a-1c44b19d377c/geofeed.csv -remarks: **************************** -remarks: * THIS OBJECT IS MODIFIED -remarks: * Please note that all data that is generally regarded as personal -remarks: * data has been removed from this object. -remarks: * To view the original object, please query the RIPE Database at: -remarks: * http://www.ripe.net/whois -remarks: **************************** - -ADD 65776783 - -inet6num: 2a0f:6280:c826::/48 -netname: FUTURE-USAGE -descr: Iridium -country: DZ -admin-c: DUMY-RIPE -tech-c: DUMY-RIPE -abuse-c: ACRO56593-RIPE -status: ASSIGNED -mnt-by: HYEHOST-MNT -org: ORG-IA2236-RIPE -created: 2026-08-19T14:41:35Z -last-modified: 2026-08-19T14:41:35Z -source: RIPE -geofeed: https://api.geofeed.space/fc79d4c7-ff20-4ed7-bc0a-1c44b19d377c/geofeed.csv -remarks: **************************** -remarks: * THIS OBJECT IS MODIFIED -remarks: * Please note that all data that is generally regarded as personal -remarks: * data has been removed from this object. -remarks: * To view the original object, please query the RIPE Database at: -remarks: * http://www.ripe.net/whois -remarks: **************************** - -ADD 65776784 - -inet6num: 2a0f:6280:c827::/48 -netname: FUTURE-USAGE -descr: Iridium -country: VN -admin-c: DUMY-RIPE -tech-c: DUMY-RIPE -abuse-c: ACRO56593-RIPE -status: ASSIGNED -mnt-by: HYEHOST-MNT -org: ORG-IA2236-RIPE -created: 2026-08-19T14:41:35Z -last-modified: 2026-08-19T14:41:35Z -source: RIPE -geofeed: https://api.geofeed.space/fc79d4c7-ff20-4ed7-bc0a-1c44b19d377c/geofeed.csv -remarks: **************************** -remarks: * THIS OBJECT IS MODIFIED -remarks: * Please note that all data that is generally regarded as personal -remarks: * data has been removed from this object. -remarks: * To view the original object, please query the RIPE Database at: -remarks: * http://www.ripe.net/whois -remarks: **************************** - -ADD 65776785 - -aut-num: AS62078 -as-name: MEASUREMENT-NETWORK-SCAN -org: ORG-TF84-RIPE -descr: MEASUREMENT-NETWORK-BGP-SCAN; See https://measurement.network/ for further information. -descr: -descr: ........................................................... -descr: . This network is part of https://measurement.network/. . -descr: . The project provides network measurement infrastructure . -descr: . for academic researchers. . -descr: . . -descr: . For an overview of currently running measurements, see . -descr: . our website or the more specific objects for individual . -descr: . networks below this prefix. . -descr: . . -descr: . AS62078 is used in active network measurements where . -descr: . BGP OPEN messages are sent to hosts publicly listening . -descr: . on tcp/179. We are using a public 16bit ASN to limit . -descr: . the potential for side effects. AS62078 should never be . -descr: . visible in the GRT. . -descr: . ------------------------------------------------------- . -descr: . To report abuse or request ranges to be excluded from . -descr: . measurements, please send an email to: . -descr: . . -descr: . abuse@measurement.network . -descr: . . -descr: . or see the ABUSE-C MAC418-RIPE. . -descr: . ------------------------------------------------------- . -descr: . If you want to actively block measurement.network, you . -descr: . can do so by blocking the following resources which are . -descr: . an exhaustive lists of the numbered resources and . -descr: . domains we use. All infrastructure is within these . -descr: . netblocks: . -descr: . . -descr: . # ASN . -descr: . AS211286 . -descr: . AS62078 . -descr: . . -descr: . # IPv4 . -descr: . 141.39.220.0/22 . -descr: . . -descr: . # IPv6 . -descr: . 2a0d:8d04::/32 . -descr: . . -descr: . # Domains . -descr: . measurement.network . -descr: ........................................................... -descr: -descr: Ignore this address: desc-asn-20260913-2230@visp.wien -admin-c: DUMY-RIPE -tech-c: DUMY-RIPE -abuse-c: AR80422-RIPE -status: ASSIGNED -mnt-by: RIPE-NCC-END-MNT -mnt-by: WYBT-MNT -mnt-by: DFN-MNT -mnt-by: MPG-MNT -created: 2014-02-21T12:28:46Z -last-modified: 2026-09-13T22:30:01Z -source: RIPE -remarks: **************************** -remarks: * THIS OBJECT IS MODIFIED -remarks: * Please note that all data that is generally regarded as personal -remarks: * data has been removed from this object. -remarks: * To view the original object, please query the RIPE Database at: -remarks: * http://www.ripe.net/whois -remarks: **************************** - From 2c26837d366ca36fc056e7cef429c2b41f2a17ea Mon Sep 17 00:00:00 2001 From: lou lecrivain Date: Tue, 22 Sep 2026 11:58:29 +0200 Subject: [PATCH 05/11] remove no-longer-used tokio-stream --- Cargo.lock | 1 - Cargo.toml | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ed3795f..9e5926f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -148,7 +148,6 @@ dependencies = [ "re-delimiter-codec", "regex", "tokio", - "tokio-stream", "tokio-test", "tokio-util", ] diff --git a/Cargo.toml b/Cargo.toml index ee1d5bd..230639c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,6 @@ pest = "2.9.0" pest_derive = "2.9.0" tokio = { version = "1.53.1", optional = true } tokio-util = { version = "0.7.19", optional = true, features = ["codec"] } -tokio-stream = { version = "0.1.19", optional = true } re-delimiter-codec = { version = "0.1.0", optional = true } regex = { version = "1.13.1", optional = true } futures-util = { version = "0.3.34", optional = true } @@ -24,7 +23,7 @@ encoding_rs = { version = "0.8.41", features = ["std"], optional = true } [features] default = [ ] -async-streaming = [ "dep:tokio-util", "dep:tokio", "dep:tokio-stream", "dep:futures-util", "dep:re-delimiter-codec", "dep:regex", "dep:chardetng", "dep:encoding_rs" ] +async-streaming = [ "dep:tokio-util", "dep:tokio", "dep:futures-util", "dep:re-delimiter-codec", "dep:regex", "dep:chardetng", "dep:encoding_rs" ] [dev-dependencies] tokio = { version = "1.53.1", features = ["fs", "macros", "rt" ] } From 02b61b400181199940cf134db6fda43f282734e1 Mon Sep 17 00:00:00 2001 From: lou lecrivain Date: Tue, 22 Sep 2026 15:33:08 +0200 Subject: [PATCH 06/11] document obscure regex + use r-string for regex --- src/streaming.rs | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/streaming.rs b/src/streaming.rs index e5c7921..4f074e3 100644 --- a/src/streaming.rs +++ b/src/streaming.rs @@ -22,7 +22,27 @@ fn new_nrtm_preparser() -> REDelimiterCodec { // ok to call unwrap here, we know this will not fail REDelimiterCodec::new_with_max_length( - Regex::new("(?R)\n[^%][^AD][^DE][^DL].*\n\n").unwrap(), + // will slice at each end of an NRTM object. + // NRTM delimiters are double new lines. + // + // we add a negative match in front of the delimiter, + // which DOES NOT match the start of an ADD/DEL operation (v2/v3). + // This is akin to negative lookbehind but ofc not as precise... + // but it's good enough for us. I am not using alternate patterns + // + // This has the effect of excluding this exact double newline separator + // and thus, we have the full NRTM message, without odd/even pairs. + // It will also not match RPSL line continuations, since the two \n + // are not right next to each other (at least one character is between them) + // + // \nDEL 65934907\n\n <- this does not match + // + // \nmulti-line: attribute\n multi-line: goes on\n <- this does not match + // + // \nremarks: ****************************\n\n <- this *does* match. end of RPSL + // + // which is what we want, since we need to cut at the end of the RPSL object. + Regex::new(r"(?R)\n[^%][^AD][^DE][^DL].*\n\n").unwrap(), MAX_CHUNK_LEN, ) } From 0e1653b23f15d612a2708734311669326f2cb4b5 Mon Sep 17 00:00:00 2001 From: lou lecrivain Date: Tue, 22 Sep 2026 16:48:00 +0200 Subject: [PATCH 07/11] improve on chunk regex + doc, now no longer matches NRTM prelude --- src/streaming.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/streaming.rs b/src/streaming.rs index 4f074e3..cce0148 100644 --- a/src/streaming.rs +++ b/src/streaming.rs @@ -25,8 +25,19 @@ fn new_nrtm_preparser() -> REDelimiterCodec { // will slice at each end of an NRTM object. // NRTM delimiters are double new lines. // + // 1st part of the regex: \n[^%].* + // + // we add a negative match in order to not match comment lines + // which are part of the prelude. sometimes NRTM preludes contain + // double newlines. + // + // + // 2nd part of the regex: \n[^AD][^DE][^DL].*\n\n + // // we add a negative match in front of the delimiter, // which DOES NOT match the start of an ADD/DEL operation (v2/v3). + // That operation will always be after the start of a newline (cannot + // have a line continuation, as opposed to RPSL). // This is akin to negative lookbehind but ofc not as precise... // but it's good enough for us. I am not using alternate patterns // @@ -35,6 +46,12 @@ fn new_nrtm_preparser() -> REDelimiterCodec { // It will also not match RPSL line continuations, since the two \n // are not right next to each other (at least one character is between them) // + // remarks: RPSL objects have at least 1 class attr and 1 attr from the class, + // so having two entire lines on the "back" part of the regex is okay. these will + // match the RPSL object partially or entirely. + // + // EXAMPLES: + // // \nDEL 65934907\n\n <- this does not match // // \nmulti-line: attribute\n multi-line: goes on\n <- this does not match @@ -42,7 +59,7 @@ fn new_nrtm_preparser() -> REDelimiterCodec { // \nremarks: ****************************\n\n <- this *does* match. end of RPSL // // which is what we want, since we need to cut at the end of the RPSL object. - Regex::new(r"(?R)\n[^%][^AD][^DE][^DL].*\n\n").unwrap(), + Regex::new(r"(?R)\n[^%].*\n[^AD][^DE][^DL].*\n\n").unwrap(), MAX_CHUNK_LEN, ) } From c43e9f26b9e9cd331ba2ef95e298005619e2ea1a Mon Sep 17 00:00:00 2001 From: lou lecrivain Date: Tue, 22 Sep 2026 16:48:25 +0200 Subject: [PATCH 08/11] bar errors from happenning with test data --- src/streaming.rs | 4 +--- tests/streaming_integration.rs | 23 ++++++++++++++--------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/streaming.rs b/src/streaming.rs index cce0148..1392db0 100644 --- a/src/streaming.rs +++ b/src/streaming.rs @@ -153,9 +153,7 @@ mod tests { rpsl: _, span: _, })) => {} // ignore - // comment at the beginning is marked as a chunk but is incomplete from the POV - // of the parser - Err(e) => println!("got error {:?}", e), + Err(e) => panic!("got error {:?}", e), Ok(None) => break, // end of stream } } diff --git a/tests/streaming_integration.rs b/tests/streaming_integration.rs index a209b00..4b46c45 100644 --- a/tests/streaming_integration.rs +++ b/tests/streaming_integration.rs @@ -19,18 +19,23 @@ async fn parse_message_stream_example() { let optional_result = stream.try_next().await; match optional_result { - Err(NRTMStreamError::Parser(ParseError::NoMatch)) - | Err(NRTMStreamError::Parser(ParseError::Incomplete)) - | Err(NRTMStreamError::Parser(ParseError::Parser(_))) - | Err(NRTMStreamError::Parser(ParseError::MalformedSerial(_, _))) - | Err(NRTMStreamError::Parser(ParseError::LeadingGarbage(_))) => { - // do nothing, all of these are retryable errors - // (we should have at least one incomplete parse error due to prelude / comment - // being chunked). ideally for each error type you'd want to log it, so that + e @ Err(NRTMStreamError::Parser(ParseError::NoMatch)) + | e @ Err(NRTMStreamError::Parser(ParseError::Incomplete)) + | e @ Err(NRTMStreamError::Parser(ParseError::Parser(_))) + | e @ Err(NRTMStreamError::Parser(ParseError::MalformedSerial(_, _))) + | e @ Err(NRTMStreamError::Parser(ParseError::LeadingGarbage(_))) => { + // all of these are retryable errors. + // ideally for each error type you'd want to log it, so that // we know some garbage data was in the stream. + panic!( + "recoverable error during consumption of NRTM stream\ + should not happen with test data. error encountered: {:?}", + e + ); } Err(error) => panic!( - "irrecoverable error during consumption of NRTM stream {:?}", + "irrecoverable error during consumption of NRTM stream\ + should not happen with test data. error encountered: {:?}", error ), Ok(None) => break, // end of stream From 7b3577174de9bafaf9d14bc156e4dfd492d8cfc7 Mon Sep 17 00:00:00 2001 From: lou lecrivain Date: Wed, 23 Sep 2026 11:12:08 +0200 Subject: [PATCH 09/11] expose max chunk length in streaming parser API --- src/lib.rs | 24 ++++++------------------ src/streaming.rs | 27 +++++++++++++++------------ tests/streaming_integration.rs | 3 ++- 3 files changed, 23 insertions(+), 31 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 3edece9..5650e2c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -87,23 +87,17 @@ where ) -> impl TryStream; } +#[cfg_attr(not(feature = "async-streaming"), derive(Default))] pub struct NRTMV3Parser { #[cfg(feature = "async-streaming")] inner_decoder: NRTMDec, } -#[cfg(feature = "async-streaming")] -impl Default for NRTMV3Parser { - fn default() -> Self { - Self::new() - } -} - impl NRTMV3Parser { #[cfg(feature = "async-streaming")] - pub fn new() -> Self { + pub fn new(max_chunk_len: usize) -> Self { Self { - inner_decoder: NRTMDec::new_v3(), + inner_decoder: NRTMDec::new_v3(max_chunk_len), } } #[cfg(not(feature = "async-streaming"))] @@ -117,23 +111,17 @@ impl NRTMParser for NRTMV3Parser { } } +#[cfg_attr(not(feature = "async-streaming"), derive(Default))] pub struct NRTMV2Parser { #[cfg(feature = "async-streaming")] inner_decoder: NRTMDec, } -#[cfg(feature = "async-streaming")] -impl Default for NRTMV2Parser { - fn default() -> Self { - Self::new() - } -} - impl NRTMV2Parser { #[cfg(feature = "async-streaming")] - pub fn new() -> Self { + pub fn new(max_chunk_len: usize) -> Self { Self { - inner_decoder: NRTMDec::new_v2(), + inner_decoder: NRTMDec::new_v2(max_chunk_len), } } #[cfg(not(feature = "async-streaming"))] diff --git a/src/streaming.rs b/src/streaming.rs index 1392db0..0bdf458 100644 --- a/src/streaming.rs +++ b/src/streaming.rs @@ -9,6 +9,7 @@ use tokio_util::codec::FramedRead; #[derive(Clone)] pub(crate) struct NRTMDec { parser: fn(&str) -> Result, + max_chunk_len: usize, } #[derive(Debug)] @@ -17,9 +18,7 @@ pub enum NRTMStreamError { Parser(ParseError), } -fn new_nrtm_preparser() -> REDelimiterCodec { - const MAX_CHUNK_LEN: usize = 131072; // 128k - +fn new_nrtm_preparser(with_max_chunk_len: usize) -> REDelimiterCodec { // ok to call unwrap here, we know this will not fail REDelimiterCodec::new_with_max_length( // will slice at each end of an NRTM object. @@ -60,27 +59,29 @@ fn new_nrtm_preparser() -> REDelimiterCodec { // // which is what we want, since we need to cut at the end of the RPSL object. Regex::new(r"(?R)\n[^%].*\n[^AD][^DE][^DL].*\n\n").unwrap(), - MAX_CHUNK_LEN, + with_max_chunk_len, ) } impl NRTMDec { - pub(crate) fn new_v2() -> Self { + pub(crate) fn new_v2(max_chunk_len: usize) -> Self { NRTMDec { parser: NRTMV2Parser::try_parse, + max_chunk_len, } } - pub(crate) fn new_v3() -> Self { + pub(crate) fn new_v3(max_chunk_len: usize) -> Self { NRTMDec { parser: NRTMV3Parser::try_parse, + max_chunk_len, } } pub(crate) fn get_stream( &mut self, reader: T, ) -> impl TryStream { - let framed_reader = FramedRead::new(reader, new_nrtm_preparser()); + let framed_reader = FramedRead::new(reader, new_nrtm_preparser(self.max_chunk_len)); let parser = self.parser; framed_reader @@ -129,12 +130,14 @@ mod tests { use tokio::fs::File; use tokio_test::io::Builder; + const TEST_MAX_CHUNK_LEN: usize = 131072; // 128K + #[tokio::test] async fn v3_charset_guesstimation_ok() { let nrtmv3_sample = File::open("./src/tests/nrtmv3_ripe_mixed_encoding_sample.txt") .await .unwrap(); - let mut decoder = NRTMDec::new_v3(); + let mut decoder = NRTMDec::new_v3(TEST_MAX_CHUNK_LEN); let mut stream = decoder.get_stream(nrtmv3_sample); let mut linear_increase_id_counter = 65934900; @@ -163,7 +166,7 @@ mod tests { #[tokio::test] async fn v3_parser_error_signalled() { - let mut decoder = NRTMDec::new_v3(); + let mut decoder = NRTMDec::new_v3(TEST_MAX_CHUNK_LEN); let mut reader = decoder.get_stream( &b"\ ADD 324876 @@ -182,7 +185,7 @@ end-of: object #[tokio::test] async fn v3_malformed_serial_signalled() { - let mut decoder = NRTMDec::new_v3(); + let mut decoder = NRTMDec::new_v3(TEST_MAX_CHUNK_LEN); let mut reader = decoder.get_stream( &b"\ # should not fit into u64 @@ -201,7 +204,7 @@ netname: TRANSPORT-NET #[tokio::test] async fn v3_no_chunks_signalled() { - let mut decoder = NRTMDec::new_v3(); + let mut decoder = NRTMDec::new_v3(TEST_MAX_CHUNK_LEN); let mut reader = decoder.get_stream( &b"\ % The RIPE Database is subject to Terms and Conditions. @@ -242,7 +245,7 @@ end-of: obj .read(nrtmv3_truncated_message) .read_error(IOError::new(ErrorKind::BrokenPipe, "connection closed")) .build(); - let mut decoder = NRTMDec::new_v3(); + let mut decoder = NRTMDec::new_v3(TEST_MAX_CHUNK_LEN); let mut reader = decoder.get_stream(ioerroring_sample); reader.try_next().await.unwrap().unwrap(); // chuck first object diff --git a/tests/streaming_integration.rs b/tests/streaming_integration.rs index 4b46c45..e855de1 100644 --- a/tests/streaming_integration.rs +++ b/tests/streaming_integration.rs @@ -9,10 +9,11 @@ use { #[cfg(feature = "async-streaming")] #[tokio::test] async fn parse_message_stream_example() { + const MAX_NRTM_CHUNK_LEN: usize = 1048576; // 1M is reasonable let nrtmv3_sample = File::open("./src/tests/nrtmv3_ripe_mixed_encoding_sample.txt") .await .unwrap(); - let mut parser = NRTMV3Parser::new(); + let mut parser = NRTMV3Parser::new(MAX_NRTM_CHUNK_LEN); let mut stream = parser.stream_from(nrtmv3_sample); loop { From 34c8c0bcf1e58f04399458f785dfc2e3b7e51fe0 Mon Sep 17 00:00:00 2001 From: lou lecrivain Date: Wed, 23 Sep 2026 11:37:04 +0200 Subject: [PATCH 10/11] document new deps --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8bf1e75..57b594d 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,8 @@ Minimum supported rust version is 1.97.1 ## Dependencies This crate is built on: - the [Pest parser](https://crates.io/crates/pest) crate -- [tokio](https://crates.io/crates/tokio), [tokio-util](https://crates.io/crates/tokio-util) and [tokio-stream](https://crates.io/crates/tokio-stream) for the async features +- mixed encoding detection and conversion is achieved through [chardetng](https://crates.io/crates/chardetng) and [encoding_rs](https://crates.io/crates/encoding_rs) +- [tokio](https://crates.io/crates/tokio), [tokio-util](https://crates.io/crates/tokio-util) and [futures-util](https://crates.io/crates/futures-util) for the async features ## Examples / How to hold it correctly 🔨 You can find some code examples in the integration tests: From ef659be81b13639f85b4feb0d87a33d7b78b73b3 Mon Sep 17 00:00:00 2001 From: lou lecrivain Date: Wed, 23 Sep 2026 15:44:21 +0200 Subject: [PATCH 11/11] bump re delimiter codec ver --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9e5926f..fc2bbbe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -220,9 +220,9 @@ dependencies = [ [[package]] name = "re-delimiter-codec" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd265cf799a0cdc9eb1417b1403990fd29aafe6992877c70f2b75d1439125e77" +checksum = "a9349d583e5c1d00627915dcb1ee3408a56e73f64abcc21529628ef97157be36" dependencies = [ "regex", "tokio-util", diff --git a/Cargo.toml b/Cargo.toml index 230639c..0500ed0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ pest = "2.9.0" pest_derive = "2.9.0" tokio = { version = "1.53.1", optional = true } tokio-util = { version = "0.7.19", optional = true, features = ["codec"] } -re-delimiter-codec = { version = "0.1.0", optional = true } +re-delimiter-codec = { version = "0.1.1", optional = true } regex = { version = "1.13.1", optional = true } futures-util = { version = "0.3.34", optional = true } chardetng = { version = "1.0.0", optional = true }