Organized indentation

cranscrape.py

@@ -60,13 +60,15 @@ urls = [
     'https://cran.r-project.org/web/packages/bizdays/index.html',
 ]
 
+
 def get_data(url):
-  res = requests.get(url)
+    res = requests.get(url)
-  m = reu.search(res.text)
+    m = reu.search(res.text)
-  if m:
+    if m:
-    return dict(cran=url, github=m.group(0), repo=m.group(1))
+        return dict(cran=url, github=m.group(0), repo=m.group(1))
-  else:
+    else:
-    return dict(cran=url, github='', repo='')
+        return dict(cran=url, github='', repo='')
+
 
 all_data = [get_data(url) for url in urls]
 df = pd.DataFrame(all_data)

parse.py

@@ -10,83 +10,84 @@ from github import Github
 # using an access token
 g = Github(os.environ['GITHUB_ACCESS_TOKEN'])
 
+
 def extract_repo(url):
-  reu = re.compile('^https://github.com/([\w-]+/[-\w\.]+)$')
+    reu = re.compile('^https://github.com/([\w-]+/[-\w\.]+)$')
-  m = reu.match(url)
+    m = reu.match(url)
-  if m:
+    if m:
-    return m.group(1)
+        return m.group(1)
-  else:
+    else:
-    return ''
+        return ''
 
 
 def get_last_commit(repo):
-  try:
+    try:
-    if repo:
+        if repo:
-      r = g.get_repo(repo)
+            r = g.get_repo(repo)
-      cs = r.get_commits()
+            cs = r.get_commits()
-      return cs[0].commit.author.date.strftime('%Y-%m-%d')
+            return cs[0].commit.author.date.strftime('%Y-%m-%d')
-    else:
+        else:
-      return ''
+            return ''
-  except:
+    except:
-    print('ERROR' + repo)
+        print('ERROR' + repo)
-    return 'error'
+        return 'error'
 
 
 class Project(Thread):
 
-  def __init__(self, match, section):
+    def __init__(self, match, section):
-    super().__init__()
+        super().__init__()
-    self._match = match
+        self._match = match
-    self.regs = None
+        self.regs = None
-    self._section = section
+        self._section = section
 
-  def run(self):
+    def run(self):
-    m = self._match
+        m = self._match
-    is_github = 'github.com' in m.group(2)
+        is_github = 'github.com' in m.group(2)
-    is_cran = 'cran.r-project.org' in m.group(2)
+        is_cran = 'cran.r-project.org' in m.group(2)
-    repo = extract_repo(m.group(2))
+        repo = extract_repo(m.group(2))
-    last_commit = get_last_commit(repo)
+        last_commit = get_last_commit(repo)
-    self.regs = dict(
+        self.regs = dict(
-      project=m.group(1),
+            project=m.group(1),
-      section=self._section,
+            section=self._section,
-      last_commit=last_commit,
+            last_commit=last_commit,
-      url=m.group(2),
+            url=m.group(2),
-      description=m.group(3),
+            description=m.group(3),
-      github=is_github,
+            github=is_github,
-      cran=is_cran,
+            cran=is_cran,
-      repo=repo
+            repo=repo
-    )
+        )
 
 
 projects = []
 
 with open('README.md', 'r', encoding='utf8') as f:
-  ret = re.compile('^(#+) (.*)$')
+    ret = re.compile('^(#+) (.*)$')
-  rex = re.compile('^\s*- \[(.*)\]\((.*)\) - (.*)$')
+    rex = re.compile('^\s*- \[(.*)\]\((.*)\) - (.*)$')
-  m_titles = []
+    m_titles = []
-  last_head_level = 0
+    last_head_level = 0
-  for line in f:
+    for line in f:
-    m = rex.match(line)
+        m = rex.match(line)
-    if m:
+        if m:
-      p = Project(m, ' > '.join(m_titles[1:]))
+            p = Project(m, ' > '.join(m_titles[1:]))
-      p.start()
+            p.start()
-      projects.append(p)
+            projects.append(p)
-    else:
-      m = ret.match(line)
-      if m:
-        hrs = m.group(1)
-        if len(hrs) > last_head_level:
-          m_titles.append(m.group(2))
         else:
-          for n in range(last_head_level - len(hrs) + 1):
+            m = ret.match(line)
-            m_titles.pop()
+            if m:
-          m_titles.append(m.group(2))
+                hrs = m.group(1)
-        last_head_level = len(hrs)
+                if len(hrs) > last_head_level:
+                    m_titles.append(m.group(2))
+                else:
+                    for n in range(last_head_level - len(hrs) + 1):
+                        m_titles.pop()
+                    m_titles.append(m.group(2))
+                last_head_level = len(hrs)
 
 while True:
-  checks = [not p.is_alive() for p in projects]
+    checks = [not p.is_alive() for p in projects]
-  if all(checks):
+    if all(checks):
-    break
+        break
 
 projects = [p.regs for p in projects]
 df = pd.DataFrame(projects)

topic.py

@@ -21,4 +21,5 @@ repos = g.search_repositories(query=f'topic:{topic}')
 for repo in repos:
     if repo.stargazers_count < 1000:
         break
-    print(repo.name, repo.stargazers_count, repo.language, repo.html_url, repo.description, repo.updated_at, repo.archived)
+    print(repo.name, repo.stargazers_count, repo.language, repo.html_url,
+          repo.description, repo.updated_at, repo.archived)