diff --git a/cran.csv b/cran.csv new file mode 100644 index 0000000..db209f1 --- /dev/null +++ b/cran.csv @@ -0,0 +1,51 @@ +cran,github,repo +https://cran.r-project.org/web/packages/xts/index.html,https://github.com/joshuaulrich/xts,joshuaulrich/xts +https://cran.r-project.org/web/packages/data.table/index.html,https://github.com/Rdatatable/data,Rdatatable/data +https://cran.r-project.org/web/packages/tseries/index.html,, +https://cran.r-project.org/web/packages/zoo/index.html,, +https://cran.r-project.org/web/packages/tis/index.html,, +https://cran.r-project.org/web/packages/tfplot/index.html,, +https://cran.r-project.org/web/packages/tframe/index.html,, +https://cran.r-project.org/web/packages/IBrokers/index.html,, +https://cran.r-project.org/web/packages/Rblpapi/index.html,https://github.com/Rblp/Rblpapi,Rblp/Rblpapi +https://cran.r-project.org/web/packages/Rbitcoin/index.html,https://github.com/jangorecki/Rbitcoin,jangorecki/Rbitcoin +https://cran.r-project.org/web/packages/GetTDData/index.html,https://github.com/msperlin/GetTDData,msperlin/GetTDData +https://cran.r-project.org/web/packages/GetHFData/index.html,, +https://cran.r-project.org/package=td,https://github.com/eddelbuettel/td,eddelbuettel/td +https://cran.r-project.org/web/packages/quantmod/index.html,https://github.com/joshuaulrich/quantmod,joshuaulrich/quantmod +https://cran.r-project.org/web/packages/fAsianOptions/index.html,, +https://cran.r-project.org/web/packages/fAssets/index.html,, +https://cran.r-project.org/web/packages/fBasics/index.html,, +https://cran.r-project.org/web/packages/fBonds/index.html,, +https://cran.r-project.org/web/packages/fExoticOptions/index.html,, +https://cran.r-project.org/web/packages/fOptions/index.html,, +https://cran.r-project.org/web/packages/fPortfolio/index.html,, +https://cran.r-project.org/web/packages/portfolio/index.html,https://github.com/dgerlanc/portfolio,dgerlanc/portfolio +https://cran.r-project.org/web/packages/portfolioSim/index.html,, +https://cran.r-project.org/web/packages/sde/index.html,, +https://cran.r-project.org/web/packages/YieldCurve/index.html,, +https://cran.r-project.org/web/packages/SmithWilsonYieldCurve/index.html,, +https://cran.r-project.org/web/packages/ycinterextra/index.html,, +https://cran.r-project.org/web/packages/AmericanCallOpt/index.html,, +https://cran.r-project.org/web/packages/VarSwapPrice/index.html,, +https://cran.r-project.org/web/packages/RND/index.html,, +https://cran.r-project.org/web/packages/LSMonteCarlo/index.html,, +https://cran.r-project.org/web/packages/OptHedging/index.html,, +https://cran.r-project.org/web/packages/tvm/index.html,, +https://cran.r-project.org/web/packages/OptionPricing/index.html,, +https://cran.r-project.org/web/packages/credule/index.html,https://github.com/blenezet/credule,blenezet/credule +https://cran.r-project.org/web/packages/derivmkts/index.html,, +https://cran.r-project.org/web/packages/PortfolioAnalytics/PortfolioAnalytics.pdf,, +https://cran.r-project.org/web/packages/backtest/index.html,, +https://cran.r-project.org/web/packages/pa/index.html,, +https://cran.r-project.org/web/packages/TTR/index.html,https://github.com/joshuaulrich/TTR,joshuaulrich/TTR +https://cran.r-project.org/web/packages/PerformanceAnalytics/index.html,https://github.com/braverock/PerformanceAnalytics,braverock/PerformanceAnalytics +https://cran.r-project.org/web/packages/tseries/index.html,, +https://cran.r-project.org/web/packages/zoo/index.html,, +https://cran.r-project.org/web/packages/xts/index.html,https://github.com/joshuaulrich/xts,joshuaulrich/xts +https://cran.r-project.org/web/packages/fGarch/index.html,, +https://cran.r-project.org/web/packages/timeSeries/index.html,, +https://cran.r-project.org/web/packages/rugarch/index.html,, +https://cran.r-project.org/web/packages/rmgarch/index.html,https://github.com/alexiosg/rmgarch,alexiosg/rmgarch +https://cran.r-project.org/web/packages/timeDate/index.html,, +https://cran.r-project.org/web/packages/bizdays/index.html,https://github.com/wilsonfreitas/R-bizdays,wilsonfreitas/R-bizdays diff --git a/cranscrape.py b/cranscrape.py new file mode 100644 index 0000000..45b0eee --- /dev/null +++ b/cranscrape.py @@ -0,0 +1,73 @@ + +import requests +import re +import pandas as pd + +reu = re.compile('https://github.com/([\w-]+/[\w-]+)') + +url = 'https://cran.r-project.org/web/packages/xts/index.html' + +urls = [ + 'https://cran.r-project.org/web/packages/xts/index.html', + 'https://cran.r-project.org/web/packages/data.table/index.html', + 'https://cran.r-project.org/web/packages/tseries/index.html', + 'https://cran.r-project.org/web/packages/zoo/index.html', + 'https://cran.r-project.org/web/packages/tis/index.html', + 'https://cran.r-project.org/web/packages/tfplot/index.html', + 'https://cran.r-project.org/web/packages/tframe/index.html', + 'https://cran.r-project.org/web/packages/IBrokers/index.html', + 'https://cran.r-project.org/web/packages/Rblpapi/index.html', + 'https://cran.r-project.org/web/packages/Rbitcoin/index.html', + 'https://cran.r-project.org/web/packages/GetTDData/index.html', + 'https://cran.r-project.org/web/packages/GetHFData/index.html', + 'https://cran.r-project.org/package=td', + 'https://cran.r-project.org/web/packages/quantmod/index.html', + 'https://cran.r-project.org/web/packages/fAsianOptions/index.html', + 'https://cran.r-project.org/web/packages/fAssets/index.html', + 'https://cran.r-project.org/web/packages/fBasics/index.html', + 'https://cran.r-project.org/web/packages/fBonds/index.html', + 'https://cran.r-project.org/web/packages/fExoticOptions/index.html', + 'https://cran.r-project.org/web/packages/fOptions/index.html', + 'https://cran.r-project.org/web/packages/fPortfolio/index.html', + 'https://cran.r-project.org/web/packages/portfolio/index.html', + 'https://cran.r-project.org/web/packages/portfolioSim/index.html', + 'https://cran.r-project.org/web/packages/sde/index.html', + 'https://cran.r-project.org/web/packages/YieldCurve/index.html', + 'https://cran.r-project.org/web/packages/SmithWilsonYieldCurve/index.html', + 'https://cran.r-project.org/web/packages/ycinterextra/index.html', + 'https://cran.r-project.org/web/packages/AmericanCallOpt/index.html', + 'https://cran.r-project.org/web/packages/VarSwapPrice/index.html', + 'https://cran.r-project.org/web/packages/RND/index.html', + 'https://cran.r-project.org/web/packages/LSMonteCarlo/index.html', + 'https://cran.r-project.org/web/packages/OptHedging/index.html', + 'https://cran.r-project.org/web/packages/tvm/index.html', + 'https://cran.r-project.org/web/packages/OptionPricing/index.html', + 'https://cran.r-project.org/web/packages/credule/index.html', + 'https://cran.r-project.org/web/packages/derivmkts/index.html', + 'https://cran.r-project.org/web/packages/PortfolioAnalytics/PortfolioAnalytics.pdf', + 'https://cran.r-project.org/web/packages/backtest/index.html', + 'https://cran.r-project.org/web/packages/pa/index.html', + 'https://cran.r-project.org/web/packages/TTR/index.html', + 'https://cran.r-project.org/web/packages/PerformanceAnalytics/index.html', + 'https://cran.r-project.org/web/packages/tseries/index.html', + 'https://cran.r-project.org/web/packages/zoo/index.html', + 'https://cran.r-project.org/web/packages/xts/index.html', + 'https://cran.r-project.org/web/packages/fGarch/index.html', + 'https://cran.r-project.org/web/packages/timeSeries/index.html', + 'https://cran.r-project.org/web/packages/rugarch/index.html', + 'https://cran.r-project.org/web/packages/rmgarch/index.html', + 'https://cran.r-project.org/web/packages/timeDate/index.html', + 'https://cran.r-project.org/web/packages/bizdays/index.html', +] + +def get_data(url): + res = requests.get(url) + m = reu.search(res.text) + if m: + return dict(cran=url, github=m.group(0), repo=m.group(1)) + else: + return dict(cran=url, github='', repo='') + +all_data = [get_data(url) for url in urls] +df = pd.DataFrame(all_data) +df.to_csv('cran.csv', index=False)