Merge pull request #65 from gbrault/master

Added text sitemap parsing

Author: sethblack

.gitignore

@@ -1,3 +1,9 @@
+# I don't want the python virtual env in github!
+venv
+
+# nor visual
+.vscode
+
 *.py[cod]
 
 # C extensions

seoanalyzer/website.py

@@ -45,11 +45,15 @@ def get_text_from_xml(self, nodelist):
     def crawl(self):
         if self.sitemap:
             page = http.get(self.sitemap)
-            xmldoc = minidom.parseString(page.data.decode('utf-8'))
-            sitemap_urls = xmldoc.getElementsByTagName('loc')
-
-            for url in sitemap_urls:
-                self.page_queue.append(self.get_text_from_xml(url.childNodes))
+            if self.sitemap.endswith('xml'):
+                xmldoc = minidom.parseString(page.data.decode('utf-8'))
+                sitemap_urls = xmldoc.getElementsByTagName('loc')
+                for url in sitemap_urls:
+                    self.page_queue.append(self.get_text_from_xml(url.childNodes))
+            elif self.sitemap.endswith('txt'):
+                sitemap_urls = page.data.decode('utf-8').split('\n')
+                for url in sitemap_urls:
+                    self.page_queue.append(url)
 
         self.page_queue.append(self.base_url)
 

test.py

@@ -0,0 +1,42 @@
+import argparse
+import inspect
+import json
+import os
+
+from jinja2 import Environment
+from jinja2 import FileSystemLoader
+from seoanalyzer import analyze
+
+
+module_path = os.path.dirname(inspect.getfile(analyze))
+
+arg_parser = argparse.ArgumentParser()
+
+arg_parser.add_argument('site', help='URL of the site you are wanting to analyze.')
+arg_parser.add_argument('-s', '--sitemap', help='URL of the sitemap to seed the crawler with.')
+arg_parser.add_argument('-f', '--output-format', help='Output format.', choices=['json', 'html', ],
+                        default='json')
+arg_parser.add_argument('-d', '--disk', help='save to disk', choices=['y', 'n', ], default='y')
+
+args = arg_parser.parse_args()
+
+output = analyze(args.site, args.sitemap)
+
+if args.output_format == 'html':
+    from jinja2 import Environment
+    from jinja2 import FileSystemLoader
+
+    env = Environment(loader=FileSystemLoader(os.path.join(module_path, 'templates')))
+    template = env.get_template('index.html')
+    output_from_parsed_template = template.render(result=output)
+    if args.disk == 'y':
+        with open("test.html", "w", encoding='utf-8') as text_file:
+            text_file.write(output_from_parsed_template)
+    else:
+        print(output_from_parsed_template)
+elif args.output_format == 'json':
+    if args.disk == 'y':
+        with open("test.json", "w", encoding='utf-8') as text_file:
+            text_file.write(json.dumps(output, indent=4, separators=(',', ': ')))
+    else:
+        print(json.dumps(output, indent=4, separators=(',', ': ')))