manpage wip

Gemfile

@@ -3,7 +3,6 @@ source "http://rubygems.org"
 gem "rake"
 gem "sinatra"
 gem "dm-core"
-gem "dm-serializer"
 gem "dm-migrations"
 gem "dm-validations"
 gem "dm-timestamps"

Gemfile.lock

@@ -14,10 +14,6 @@ GEM
     dm-postgres-adapter (1.1.0)
       dm-do-adapter (~> 1.1.0)
       do_postgres (~> 0.10.2)
-    dm-serializer (1.1.0)
-      dm-core (~> 1.1.0)
-      fastercsv (~> 1.5.4)
-      json (~> 1.4.6)
     dm-sqlite-adapter (1.1.0)
       dm-do-adapter (~> 1.1.0)
       do_sqlite3 (~> 0.10.2)
@@ -29,7 +25,6 @@ GEM
       data_objects (= 0.10.6)
     do_sqlite3 (0.10.6)
       data_objects (= 0.10.6)
-    fastercsv (1.5.4)
     heroku (2.7.0)
       launchy (>= 0.3.2)
       rest-client (~> 1.6.1)
@@ -38,7 +33,6 @@ GEM
     httparty (0.8.0)
       multi_json
       multi_xml
-    json (1.4.6)
     launchy (2.0.5)
       addressable (~> 2.2.6)
     mime-types (1.16)
@@ -68,7 +62,6 @@ DEPENDENCIES
   dm-core
   dm-migrations
   dm-postgres-adapter
-  dm-serializer
   dm-sqlite-adapter
   dm-timestamps
   dm-validations

Rakefile

@@ -1,4 +1,4 @@
-require './lib/version'
+require './lib/db'
 
 desc "This task is called by the Heroku cron add-on"
 task :cron do

app.rb

@@ -1,7 +1,6 @@
 require 'sinatra'
 
-require './lib/version'
-
+require './lib/db'
 
 class GitApp < Sinatra::Base
 

lib/db.rb

@@ -0,0 +1,11 @@
+require 'dm-core'
+require 'dm-migrations'
+require 'dm-validations'
+require 'dm-timestamps'
+
+DataMapper.setup(:default, ENV['DATABASE_URL'] || "sqlite3://#{Dir.pwd}/local.db")
+
+require 'lib/version'
+require 'lib/doc'
+
+DataMapper.auto_upgrade!

lib/doc.rb

@@ -0,0 +1,24 @@
+class DocFile
+  include DataMapper::Resource
+  has n, :docs
+  property :id,          Serial
+  property :name,        String, :length => 250, :key => true
+end
+
+class Doc
+  include DataMapper::Resource
+  belongs_to :doc_file
+  belongs_to :doc_version
+  property :id,          Serial
+  property :doc,         Text
+end
+
+class DocVersion
+  include DataMapper::Resource
+  has n, :docs
+  property :id,          Serial
+  property :version,     String, :length => 250
+  property :commit_sha,  String
+  property :tree_sha,    String
+  property :created_at,  DateTime
+end

lib/version.rb

@@ -1,13 +1,5 @@
 require 'httparty'
 
-require 'dm-core'
-require 'dm-serializer/to_json'
-require 'dm-migrations'
-require 'dm-validations'
-require 'dm-timestamps'
-
-DataMapper.setup(:default, ENV['DATABASE_URL'] || "sqlite3://#{Dir.pwd}/local.db")
-
 class Version
   include DataMapper::Resource
   include HTTParty
@@ -48,4 +40,3 @@ class Version
 
 end
 
-DataMapper.auto_upgrade!

script/fill_docs.rb

@@ -0,0 +1,74 @@
+require 'httparty'
+require 'lib/db'
+
+class Documentor
+  include HTTParty
+
+  base_uri 'https://api.github.com'
+  default_params :output => 'json'
+
+  def self.populate(full = false)
+    html_sha = get("/repos/schacon/git/git/refs/heads/html").parsed_response['object']['sha']
+    commits = get_commits(html_sha, full)
+    populate_docs(commits)
+  end
+
+  def self.populate_docs(commits)
+    p commits.first
+    commits.each do |version, sha, tree, date|
+      puts "V: #{version}"
+      doc = DocVersion.first(:version => version)
+      if !doc
+        doc_version = DocVersion.create(:version => version, :commit_sha => sha, :tree_sha => tree, :created_at => date)
+        # get a list of all the html pages
+        tree = get("/repos/schacon/git/git/trees/#{tree}").parsed_response['tree']
+        tree.each do |entry|
+          if (entry['path'] =~ /\.html$/) && (entry['mode'] == '100644')
+            name = entry['path'].gsub('.html', '')
+            blob_sha = entry['sha']
+            puts name
+            puts blob_sha
+            # create a DocFile if it doesn't exist
+            doc_file = DocFile.first_or_create(:name => name)
+
+            # create a doc of the contents
+            html = get("/repos/schacon/git/git/blobs/#{blob_sha}", :headers => {'Accept' => 'application/vnd.github-blob.raw'}).body
+            doc = Doc.create(:doc => html, :doc_file => doc_file, :doc_version => doc_version)
+          end
+        end
+      end
+    end
+  end
+
+  def self.get_commits(sha, full)
+    commits = []
+    cpage, sha = get_commits_page(sha)
+    commits += cpage
+    if full
+      while cpage.size > 0
+        cpage, sha = get_commits_page(sha)
+        commits += cpage
+      end
+    end
+    commits
+  end
+
+  def self.get_commits_page(sha)
+    last_sha = nil
+    shas = []
+    commits = get("/repos/schacon/git/commits", :query => {:sha => sha, :per_page => 100}).parsed_response
+    commits.each do |commit|
+      message = commit['commit']['message'].split("\n").first
+      if m = /^Autogenerated HTML docs for v([\d\.]+?)$/.match(message)
+        puts m[1]
+        shas << [m[1], commit['sha'], commit['commit']['tree']['sha'], commit['commit']['author']['date']]
+      end
+      last_sha = commit['sha']
+    end
+    [shas, last_sha]
+  end
+
+end
+
+Documentor.populate
+