From 03489d5944e68d9aa1aa2c639e17e9474279126a Mon Sep 17 00:00:00 2001 From: Kyle Bowman Date: Mon, 20 Jan 2025 16:13:32 -0500 Subject: [PATCH] finish refactoring cli fns to use nomlist --- src/nom/base.py | 7 +++ src/nom/entry.py | 12 +--- src/nom/feed.py | 62 +++++++++---------- src/nom/main.py | 11 ++-- tests/data/entry_multi.csv | 6 +- tests/data/entry_single.csv | 4 +- tests/data/feedlist.csv | 5 +- .../data/feeds/simonwillison.net-tags-2d.atom | 34 ++++++++++ .../simonwillison.net-tags-trafficserver.atom | 22 +++++++ tests/test_entry.py | 5 +- tests/test_feed.py | 22 +++++-- 11 files changed, 126 insertions(+), 64 deletions(-) create mode 100644 tests/data/feeds/simonwillison.net-tags-2d.atom create mode 100644 tests/data/feeds/simonwillison.net-tags-trafficserver.atom diff --git a/src/nom/base.py b/src/nom/base.py index 9897c9f..bcbbc41 100644 --- a/src/nom/base.py +++ b/src/nom/base.py @@ -46,11 +46,18 @@ class NomList: def __eq__(self, other): return self.items == other.items + def __iter__(self): + return self.items.__iter__() + def __len__(self): return len(self.items) def update(self, other): self.items.update(other.items) + + def to_stdout(self): + for item in self.items: + print(item.to_str(delimeter="\t")) # NOTE: To get the interface that I want (i.e `from_csv(path)`) diff --git a/src/nom/entry.py b/src/nom/entry.py index c3eb15d..3ef20d8 100644 --- a/src/nom/entry.py +++ b/src/nom/entry.py @@ -9,10 +9,9 @@ from nom.base import NomList, NomListItem # TODO: Use proper types, not strings. (Pydantic?) @dataclass class EntryListItem(NomListItem): - id_: str title: str - url: str = "" - date: Optional[str] = "test" + url: str + date: str feed_url: Optional[str] = "" feed_alias: Optional[str] = "" viewed: Optional[bool] = "False" @@ -26,9 +25,4 @@ class EntryList(NomList): @classmethod def from_csv(cls, path: Path): - return super().from_csv(path, EntryListItem) - - # I think I like from_feed/update in place of update_from_feeds - @classmethod - def from_feed(cls, feed): - pass \ No newline at end of file + return super().from_csv(path, EntryListItem) \ No newline at end of file diff --git a/src/nom/feed.py b/src/nom/feed.py index ba3e70b..11a7f7c 100644 --- a/src/nom/feed.py +++ b/src/nom/feed.py @@ -6,53 +6,49 @@ from dataclasses import dataclass import feedparser import requests -from nom.entry import EntryListItem +from nom.entry import EntryList, EntryListItem from nom.utils import url2filename from nom.base import NomListItem, NomList class Feed: - def __init__(self, url: str): - d = feedparser.parse(url) - self.d = d - self.name = d.feed.title - self.url = url # how is this different from d.feed.link? - self.entries : list[EntryListItem] = [ - EntryListItem( - e.title, - e.link, - e.updated - ) for e in d.entries] - - def to_stdout(self): - for entry in self.entries: - if entry: - print(entry.to_str()) + def __init__(self, feedparsable): + d = feedparser.parse(feedparsable) + self.name = d.feed.title + self.url = d.feed.link # how is this different from d.feed.link? + self.entries = d.entries + + def to_entrylist(self)->EntryList: + items = [] + for e in self.entries: + entry = EntryListItem( + e.title, e.link, e.updated, + self.url, "no alias", "False", "no summary") + items.append(entry) + return EntryList(items=items) + @dataclass class FeedListItem(NomListItem): - id_: str url: str - alias: str - # Where do fetch/parse belong? - def fetch(self, url): - pass + def __hash__(self): + return hash(self.url) + + def to_feed(self): + return Feed(self.url) - def parse(self, url)->list[EntryListItem]: + def fetch_feed(self): pass class FeedList(NomList): - # TODO: Make this follow the NomList pattern - def __init__(self, name, urls): - self.name = name - self.urls = urls - #self.feeds = [] - - # TODO: Rewrite this with CSV and FeedListItem parsing + @classmethod + def from_csv(cls, path: Path): + return super().from_csv(path, FeedListItem) + @classmethod def from_file(cls, file: Path): with open(file, 'r') as f: @@ -63,11 +59,11 @@ class FeedList(NomList): if not os.path.exists(save_dir): os.makedirs(save_dir) - for url in self.urls: - filename = url2filename(url) + for flitem in self.items: + filename = url2filename(flitem.url) path = save_dir / filename with open(path, 'w') as f: # TODO: URL Error Handling - r = requests.get(url) + r = requests.get(flitem.url) f.write(r.text) print(f"{path} updated") \ No newline at end of file diff --git a/src/nom/main.py b/src/nom/main.py index 24acb69..c697bb4 100644 --- a/src/nom/main.py +++ b/src/nom/main.py @@ -2,24 +2,25 @@ from pathlib import Path from nom.utils import url2filename, NomError from nom.feed import Feed, FeedList +from nom.entry import EntryList from nom.cli import cli # Globals. Sue me. FEED_CACHE=Path.home() / ".cache" / "nom" / "feeds" FEED_LIST=Path.home() / ".local" / "share" / "nom" / "feedlist" / "default" -# TODO: Need to append feeds to one another (and save to entrylist) # TODO: Flesh out CLI. def main(): parser = cli() args = parser.parse_args() # Direct Logic - feedlist=FeedList.from_file(FEED_LIST) + feedlist=FeedList.from_csv(FEED_LIST) if args.command == "entry" and args.entry_command == "show": - for url in feedlist.urls: - feed=Feed(str(FEED_CACHE / url2filename(url))) - feed.to_stdout() + elist=EntryList() + for flitem in feedlist: + elist.update(flitem.to_feed().to_entrylist()) + elist.to_stdout() elif args.command == "feed" and args.feed_command == "update": feedlist.fetch_feeds(FEED_CACHE) else: diff --git a/tests/data/entry_multi.csv b/tests/data/entry_multi.csv index cfa5b87..bb54fee 100644 --- a/tests/data/entry_multi.csv +++ b/tests/data/entry_multi.csv @@ -1,3 +1,3 @@ -id_|title|url|date|feed_url|feed_alias|viewed|summary -2|Entry One|https://path/to/entry2.html|test|||False| -3|Entry Two|https://path/to/entry3.html|test|||True| +title|url|date|feed_url|feed_alias|viewed|summary +Entry One|https://path/to/entry2.html|date|https://path/to/feed.atom|feed1|False| +Entry Two|https://path/to/entry3.html|date|https://path/to/feed.atom|feed2|True| diff --git a/tests/data/entry_single.csv b/tests/data/entry_single.csv index c3d8f78..c90648f 100644 --- a/tests/data/entry_single.csv +++ b/tests/data/entry_single.csv @@ -1,2 +1,2 @@ -id_|title|url|date|feed_url|feed_alias|viewed|summary -1|Entry One|https://path/to/entry1.html|test|||False| +title|url|date|feed_url|feed_alias|viewed|summary +Entry One|https://path/to/entry1.html|date|https://path/to/feed.atom|no alias|False|no summary diff --git a/tests/data/feedlist.csv b/tests/data/feedlist.csv index f0c612c..3270b4e 100644 --- a/tests/data/feedlist.csv +++ b/tests/data/feedlist.csv @@ -1,2 +1,3 @@ -https://simonwillison.net/atom/everything/ -https://jvns.ca/atom.xml \ No newline at end of file +url +https://simonwillison.net/tags/2d.atom +https://simonwillison.net/tags/trafficserver.atom \ No newline at end of file diff --git a/tests/data/feeds/simonwillison.net-tags-2d.atom b/tests/data/feeds/simonwillison.net-tags-2d.atom new file mode 100644 index 0000000..e3627db --- /dev/null +++ b/tests/data/feeds/simonwillison.net-tags-2d.atom @@ -0,0 +1,34 @@ + +Simon Willison's Weblog: 2dhttp://simonwillison.net/2008-03-16T16:24:38+00:00Simon Willisondojox.gfx demos2008-03-16T16:24:38+00:002008-03-16T16:24:38+00:00https://simonwillison.net/2008/Mar/16/dojox/#atom-tag + +<p><strong><a href="http://archive.dojotoolkit.org/nightly/dojotoolkit/dojox/gfx/demos/">dojox.gfx demos</a></strong></p> +Impressive demos of the Dojo 2D drawing APIs—these need to be linked from the dojo site, it took me quite a while to find them. + + + <p>Tags: <a href="https://simonwillison.net/tags/dojo">dojo</a>, <a href="https://simonwillison.net/tags/dojox">dojox</a>, <a href="https://simonwillison.net/tags/gfx">gfx</a>, <a href="https://simonwillison.net/tags/javascript">javascript</a>, <a href="https://simonwillison.net/tags/drawing">drawing</a>, <a href="https://simonwillison.net/tags/2d">2d</a></p> + + + +Create cross browser vector graphics2006-12-20T00:42:30+00:002006-12-20T00:42:30+00:00https://simonwillison.net/2006/Dec/20/gfx/#atom-tag + +<p><strong><a href="http://www.thinkvitamin.com/features/design/create-cross-browser-vector-graphics">Create cross browser vector graphics</a></strong></p> +An accessible introduction to dojo.gfx, a powerful 2D drawing API built on SVG and VML. + + + <p>Tags: <a href="https://simonwillison.net/tags/svg">svg</a>, <a href="https://simonwillison.net/tags/drawing">drawing</a>, <a href="https://simonwillison.net/tags/dojo">dojo</a>, <a href="https://simonwillison.net/tags/2d">2d</a>, <a href="https://simonwillison.net/tags/vml">vml</a>, <a href="https://simonwillison.net/tags/javascript">javascript</a></p> + + + +Dojo 0.4 release notes2006-10-23T00:39:10+00:002006-10-23T00:39:10+00:00https://simonwillison.net/2006/Oct/23/dojo/#atom-tag + +<p><strong><a href="http://dojo.jot.com/WikiHome/Release0Point4">Dojo 0.4 release notes</a></strong></p> +GFX (a 2D drawing API) is awesome; dojo.html.metrics looks extremely useful, and onDomLoad is always nice. + + <p><small></small>Via <a href="http://blog.dojotoolkit.org/2006/10/22/dojo-040-release-candidate">The Dojo Blog</a></small></p> + + + <p>Tags: <a href="https://simonwillison.net/tags/dojo">dojo</a>, <a href="https://simonwillison.net/tags/2d">2d</a>, <a href="https://simonwillison.net/tags/drawing">drawing</a>, <a href="https://simonwillison.net/tags/javascript">javascript</a></p> + + + + \ No newline at end of file diff --git a/tests/data/feeds/simonwillison.net-tags-trafficserver.atom b/tests/data/feeds/simonwillison.net-tags-trafficserver.atom new file mode 100644 index 0000000..288890a --- /dev/null +++ b/tests/data/feeds/simonwillison.net-tags-trafficserver.atom @@ -0,0 +1,22 @@ + +Simon Willison's Weblog: trafficserverhttp://simonwillison.net/2009-11-01T12:15:27+00:00Simon WillisonTraffic Server2009-11-01T12:15:27+00:002009-11-01T12:15:27+00:00https://simonwillison.net/2009/Nov/1/trafficserver/#atom-tag + +<p><strong><a href="http://www.mnot.net/blog/2009/10/30/traffic_server">Traffic Server</a></strong></p> +Mark Nottingham explains the release of Traffic Server, a new Apache Incubator open source project donated by Yahoo! using code originally developed at Inktomi around a decade ago. Traffic Server is a HTTP proxy/cache, similar to Squid and Varnish (though Traffic Server acts as both a forward and reverse proxy, whereas Varnish only handles reverse). + + + <p>Tags: <a href="https://simonwillison.net/tags/trafficserver">trafficserver</a>, <a href="https://simonwillison.net/tags/yahoo">yahoo</a>, <a href="https://simonwillison.net/tags/inktomi">inktomi</a>, <a href="https://simonwillison.net/tags/mark-nottingham">mark-nottingham</a>, <a href="https://simonwillison.net/tags/open-source">open-source</a>, <a href="https://simonwillison.net/tags/apache">apache</a>, <a href="https://simonwillison.net/tags/http">http</a>, <a href="https://simonwillison.net/tags/cache">cache</a>, <a href="https://simonwillison.net/tags/proxy">proxy</a>, <a href="https://simonwillison.net/tags/squid">squid</a>, <a href="https://simonwillison.net/tags/varnish">varnish</a></p> + + + +Yahoo! proposal to open source "Traffic Server" via the ASF2009-07-07T12:37:02+00:002009-07-07T12:37:02+00:00https://simonwillison.net/2009/Jul/7/trafficserver/#atom-tag + +<p><strong><a href="http://wiki.apache.org/incubator/TrafficServerProposal">Yahoo! proposal to open source &quot;Traffic Server&quot; via the ASF</a></strong></p> +Traffic Server is a “fast, scalable and extensible HTTP/1.1 compliant caching proxy server” (presumably equivalent to things like Squid and Varnish) originally acquired from Inktomi and developed internally at Yahoo! for the past three years, which has been benchmarked handling 35,000 req/s on a single box. No source code yet but it looks like the release will arrive pretty soon. + + + <p>Tags: <a href="https://simonwillison.net/tags/trafficserver">trafficserver</a>, <a href="https://simonwillison.net/tags/yahoo">yahoo</a>, <a href="https://simonwillison.net/tags/open-source">open-source</a>, <a href="https://simonwillison.net/tags/caching">caching</a>, <a href="https://simonwillison.net/tags/proxy">proxy</a>, <a href="https://simonwillison.net/tags/squid">squid</a>, <a href="https://simonwillison.net/tags/varnish">varnish</a>, <a href="https://simonwillison.net/tags/apache">apache</a>, <a href="https://simonwillison.net/tags/asf">asf</a></p> + + + + \ No newline at end of file diff --git a/tests/test_entry.py b/tests/test_entry.py index e5cc8d2..a769ead 100644 --- a/tests/test_entry.py +++ b/tests/test_entry.py @@ -4,6 +4,7 @@ from copy import copy import pytest from nom.entry import EntryList, EntryListItem +from test_feed import feedlist @pytest.fixture @@ -30,9 +31,6 @@ def test_elist_to_from_csv_idempotency(elist_single,tmp_path): remade = EntryList.from_csv(path) assert remade == elist_single -def test_from_feed(): - pass - def test_eli_to_from_dict_idempotency(elist_item): remade = elist_item.from_dict(elist_item.to_dict()) assert remade == elist_item @@ -46,4 +44,3 @@ def test_elist_addition(elist_multi, elist_single): sum_ = elist_multi + elist_single assert len(sum_) == len(elist_multi) + len(elist_single) assert isinstance(sum_,EntryList) - diff --git a/tests/test_feed.py b/tests/test_feed.py index b4176da..349276e 100644 --- a/tests/test_feed.py +++ b/tests/test_feed.py @@ -1,10 +1,20 @@ from pathlib import Path +import pytest -from nom.feed import * +from nom.feed import FeedList +from nom.entry import EntryList -FEED_LIST=Path(__file__).parent / "data" / "feedlist.csv" +@pytest.fixture +def feedlist(): + path = Path(__file__).parent / "data" / "feedlist.csv" + return FeedList.from_csv(path) -def test_from_file(): - feedlist = FeedList.from_file(FEED_LIST) - assert len(feedlist.urls) == 2 - assert "jvn" in feedlist.urls[1] \ No newline at end of file +def test_flist_from_csv(feedlist): + assert len(feedlist) == 2 + #assert "https://simonwillison.net/tags/trafficserver.atom" in feedlist + +def test_to_entrylist(feedlist): + elist = EntryList() + for flitem in feedlist: + elist.update(flitem.to_feed().to_entrylist()) + assert len(elist) == 5 \ No newline at end of file -- 2.39.5