Initial revision of "nyaci"

9 files changed, 633 insertions, 0 deletions

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..54b7261
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+config.rb
diff --git a/COPYING b/COPYING
new file mode 100644
index 0000000..2dd3195
--- /dev/null
+++ b/COPYING
@@ -0,0 +1,19 @@
+Copyright (c) 2016-2017 Kazuki Yamaguchi <k@rhe.jp>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..250061d
--- /dev/null
+++ b/README.md
@@ -0,0 +1,41 @@
+nyaci
+=====
+
+CI environment for ci.rhe.jp.
+
+Work in progress.
+
+Usage
+-----
+
+    # Edit the configuration
+    cp config.rb.example config.rb
+    $EDITOR config.rb
+
+    # Start the Sinatra-based web application
+    ruby webapp.rb
+
+    # Request a build
+    curl -d secret=<secret> -d branch=<refname> https://ci.rhe.jp/<project>
+
+You may want to kick it from the git hooks (post-receive):
+
+    #!/bin/sh
+
+    while read oldrev newrev refname
+    do
+      if test "$newrev" = "0000000000000000000000000000000000000000"
+      then
+        # Removing a branch
+        continue
+      fi
+
+      branch=$(git rev-parse --symbolic --abbrev-ref "$refname")
+      echo -n "Ref '$refname': "
+      curl -s -d secret=unya -d branch="$branch" https://ci.rhe.jp/project
+    done
+
+License
+-------
+
+nyaci is licensed under the MIT license. See COPYING.
diff --git a/config.rb.example b/config.rb.example
new file mode 100644
index 0000000..174c6ed
--- /dev/null
+++ b/config.rb.example
@@ -0,0 +1,74 @@
+NyaConfig.configure(
+  parallelism: 8,
+  basedir: "/ci",
+  secret: "unya",
+  webroot: "https://ci.rhe.jp",
+  notification: {
+    email: {
+      smtp_server: "smtp://walnut.intra.rhe.jp:25",
+      from: "nyaci@ci.rhe.jp",
+      to: "k@rhe.jp",
+    },
+  },
+  modules: {
+    ruby: {
+      "2.3": {
+        prefix: "/usr"
+      },
+      trunk: {
+        prefix: "/opt/ruby-trunk"
+      },
+    },
+    openssl: {
+      "openssl-1.1.0-no-ec": {
+        prefix: "/ci/modules/openssl/openssl-1.1.0-no-ec",
+      },
+      "openssl-1.1.0-no-engine": {
+        prefix: "/ci/modules/openssl/openssl-1.1.0-no-engine",
+      },
+      "openssl-1.1.0": {
+        prefix: "/ci/modules/openssl/openssl-1.1.0",
+      },
+      "openssl-1.0.2": {
+        prefix: "/ci/modules/openssl/openssl-1.0.2",
+      },
+      "openssl-1.0.1": {
+        prefix: "/ci/modules/openssl/openssl-1.0.1",
+      },
+    }
+  },
+  projects: {
+    "ruby-openssl": {
+      git: "git://git.intra.rhe.jp/ruby-openssl.git",
+      matrix: {
+        ruby: [
+          "2.3",
+          "trunk",
+        ],
+        openssl: [
+          "openssl-1.1.0",
+          "openssl-1.0.2",
+          "openssl-1.0.1",
+          "openssl-1.1.0-no-ec",
+          "openssl-1.1.0-no-engine",
+        ]
+      },
+      build_proc: -> (t, ruby, openssl) {
+        ENV["GEM_HOME"] = File.expand_path("tmp/gems")
+        ENV["PATH"] = [
+          File.expand_path("tmp/gems/bin"),
+          File.expand_path("bin", openssl.prefix),
+          File.expand_path("bin", ruby.prefix),
+          ENV["PATH"]
+        ].join(":")
+
+        t.run "ruby -v"
+        t.run "openssl version"
+        t.run "gem install --no-user-install -N rake-compiler test-unit"
+
+        t.run "rake compile -- --with-openssl-dir=#{t.s openssl.prefix} --enable-debug"
+        t.run "rake test TESTOPTS=-v OSSL_MDEBUG=1"
+      }
+    },
+  }
+)
diff --git a/notification.rb b/notification.rb
new file mode 100644
index 0000000..f8dd526
--- /dev/null
+++ b/notification.rb
@@ -0,0 +1,71 @@
+require "net/smtp"
+require "uri"
+require "time"
+
+class Notification
+  def initialize(email: nil)
+    @list = []
+    @list << Email.new(email) if email
+  end
+
+  def publish(*args)
+    # FIXME
+    @list.each { |t| t.publish(*args) }
+  end
+
+  class Email
+    def initialize(smtp_server:, from:, to:)
+      @smtp_server = URI.parse(smtp_server)
+      @from = from
+      @to = to
+    end
+
+    def finish(subject, body, id)
+      Net::SMTP.start(@smtp_server.host, @smtp_server.port) { |smtp|
+        smtp.send_message(<<~EOF, @from, @to)
+          From: nyaci <#{@from}>
+          To: #{@to}
+          Date: #{Time.now.rfc2822}
+          Subject: #{subject.delete("\r\n")}
+          Message-Id: <nyaci-#{id}@#{@smtp_server.host}>
+
+          #{body.gsub(/(?<!\r)\n/, "\r\n")}
+        EOF
+      }
+    end
+
+    def publish(project, jobid, results)
+      prefix = "[NyaCI] [#{project}/#{jobid}]"
+
+      results.sort_by! { |id, result| id }
+      failed = results.count { |id, result| !result }
+      if failed == 0
+        subject = prefix + " success"
+        body = <<~EOM
+          All test cases passed successfully:
+
+          #{results.map { |id, res| "- #{id}" }.join("\n")}
+
+          #{NyaConfig.webroot + "/" + project + "/" + jobid}
+        EOM
+      else
+        subject = prefix + " #{failed}/#{results.size} failed"
+        body = <<~EOM
+          The following test cases failed:
+
+          #{results.select { |id, res| !res }
+              .map { |id, res| "- #{id}" }.join("\n")}
+
+          The rest cases passed:
+
+          #{results.select { |id, res| res }
+              .map { |id, res| "- #{id}" }.join("\n")}
+
+          #{NyaConfig.webroot + "/" + project + "/" + jobid}
+        EOM
+      end
+
+      finish(subject, body, jobid)
+    end
+  end
+end
diff --git a/nyabuild.rb b/nyabuild.rb
new file mode 100644
index 0000000..0428c1d
--- /dev/null
+++ b/nyabuild.rb
@@ -0,0 +1,164 @@
+require "fileutils"
+require "pathname"
+require "shellwords"
+require "time"
+require "open3"
+require "tmpdir"
+require_relative "nyacommon"
+require_relative "config"
+
+class NyaLogger
+  attr_reader :io
+
+  def initialize(path)
+    @io = File.open(path, "w")
+    @tag_map = {}
+    @begin_time = Time.now
+    puts "= begin # #{NyaUtils.flat_time}"
+  end
+
+  def item(tag)
+    if @tag_map[tag]
+      tag += "(#{@tag_map[tag] += 1})"
+    else
+      @tag_map[tag] = 1
+    end
+
+    puts "== #{tag} # #{NyaUtils.flat_time}"
+    yield
+    puts
+  end
+
+  def log(str)
+    puts "+ #{str}"
+  end
+
+  def warn(str)
+    puts "++ #{str}"
+  end
+
+  def finish(success)
+    puts "= end # #{NyaUtils.flat_time}"
+    puts "RESULT=#{success ? "success" : "failure"}"
+    puts "TIME=#{Time.now - @begin_time}"
+    @io.close
+  end
+
+  private def puts(str = "")
+    @io.puts(str)
+    @io.flush
+  end
+end
+
+class NyaBuildTarget
+  def initialize(build, path, logger)
+    @build = build
+    @path = path
+    @tag_map = {}
+    @logger = logger
+  end
+
+  private def run0(cmd, **opts)
+    @logger.log(cmd)
+    status = Open3.popen2e(cmd, **opts) { |stdin, mergedout, wait_thread|
+      stdin.close
+      FileUtils.copy_stream(mergedout, @logger.io)
+      wait_thread.value
+    }
+    unless status.success?
+      @logger.warn(status.inspect)
+      raise status.inspect
+    end
+  end
+
+  def run(cmd, **opts)
+    @logger.item(cmd.shellsplit[0]) {
+      run0(cmd, **opts)
+    }
+  end
+
+  def project() @build.project end
+
+  def cdir
+    Pathname.getwd.relative_path_from(Pathname.new(@path)).to_s
+  end
+
+  def s(str)
+    Shellwords.shellescape(str)
+  end
+
+  def git(repo, dest, branch:)
+    @logger.item("git/clone") {
+      run0 "git clone --depth 1 -q -b #{s branch} #{s repo} #{s dest}"
+      run0 "git -C #{s dest} log --max-count=1"
+    }
+  end
+
+  def self.start(build, confs, logger)
+    Dir.mktmpdir(nil, NyaConfig.tmpdir) { |path|
+      FileUtils.cd(path) {
+        t = new(build, path, logger)
+        t.git(t.project.git, build.project_id, branch: build.branch)
+        FileUtils.cd(build.project_id) {
+          t.project.build_proc.call(t, *confs)
+        }
+      }
+    }
+  end
+end
+
+class NyaBuild
+  attr_reader :project, :project_id, :branch
+
+  def initialize(project_id, branch, jobid = "#{NyaUtils.flat_time}-#{branch}")
+    @project = NyaConfig.project(project_id.intern)
+    @project_id = project_id
+    @branch = branch
+    @jobid = NyaUtils.normalize_str(jobid)
+
+    @jobdir = File.join(NyaConfig.datadir, @project_id, @jobid)
+    FileUtils.mkdir_p(@jobdir)
+
+    @results = []
+  end
+
+  def run
+    patterns = @project.matrix.dup
+
+    NyaConfig.parallelism.times.map { |x|
+      Thread.start {
+        while ary = patterns.pop
+          do_pattern(ary)
+        end
+      }
+    }.map(&:join)
+
+    NyaConfig.notification.publish(@project_id, @jobid, @results)
+  end
+
+  def do_pattern(ary)
+    pattern_tag = ary.join("_")
+
+    logger = newlogger(pattern_tag)
+    pid = fork { NyaBuildTarget.start(self, ary, logger) }
+    _, status = Process.wait2 pid
+
+    logger.finish(status.success?)
+
+    @results << [pattern_tag, status.success?]
+  end
+
+  def newlogger(pattern_tag)
+    NyaLogger.new(File.join(@jobdir, "#{pattern_tag}.log"))
+  end
+end
+
+if $0 == __FILE__
+  project_id, branch = ARGV[0], ARGV[1]
+  unless project_id && branch
+    warn "usage: nyabuild.rb <project_id> <branch>"
+    abort
+  end
+
+  NyaBuild.new(project_id, branch).run
+end
diff --git a/nyaci.service b/nyaci.service
new file mode 100644
index 0000000..aef2937
--- /dev/null
+++ b/nyaci.service
@@ -0,0 +1,12 @@
+[Unit]
+Description=NyaCI Web UI
+After=network.target
+
+[Service]
+User=ci
+Group=ci
+WorkingDirectory=/ci/nyaci
+ExecStart=/usr/bin/ruby webapp.rb -eproduction -o0.0.0.0 -p9292
+
+[Install]
+WantedBy=multi-user.target
diff --git a/nyacommon.rb b/nyacommon.rb
new file mode 100644
index 0000000..04b2ac8
--- /dev/null
+++ b/nyacommon.rb
@@ -0,0 +1,126 @@
+require "forwardable"
+require_relative "notification"
+
+module NyaConfig
+  class << self
+    attr_reader :parallelism
+    attr_reader :basedir
+    attr_reader :secret
+    attr_reader :webroot
+    attr_reader :notification
+    attr_reader :modules
+    attr_reader :projects
+
+    def configure(parallelism: 8,
+                  basedir:,
+                  secret:,
+                  webroot:,
+                  notification:,
+                  modules:,
+                  projects:)
+      @parallelism = parallelism
+      @basedir = basedir
+      @secret = secret
+      @webroot = webroot
+      @notification = Notification.new(notification)
+      @modules = modules.map { |mn, s|
+        [mn, s.map { |v, opts| [v, DependencyModule.new(mn, v, opts)] }.to_h]
+      }.to_h
+      @projects = projects.transform_values { |p| Project.new(p) }
+    end
+
+    def datadir() File.join(basedir, "data") end
+    def tmpdir() File.join(basedir, "tmp") end
+    def module(k, v) modules.fetch(k).fetch(v) end
+    def project(n) projects.fetch(n) end
+  end
+
+  class Project
+    attr_reader :git, :matrix, :build_proc
+
+    def initialize(git:, matrix:, build_proc:)
+      @git = git
+      deps = matrix.map { |name, vars|
+        vars.map { |v| NyaConfig.module(name, v.intern) }
+      }
+      @matrix = deps[0].product(*deps[1..-1])
+      @build_proc = build_proc
+    end
+  end
+
+  class DependencyModule
+    attr_reader :module_name, :version
+
+    def initialize(module_name, version, **opts)
+      @module_name = module_name
+      @version = version.intern
+      @opts = opts
+    end
+
+    def method_missing(n, *args)
+      # FIXME
+      super unless args.empty?
+      @opts[n] or super
+    end
+
+    def to_s
+      "#{module_name}-#{version}"
+    end
+  end
+end
+
+module NyaUtils
+  module_function
+  def valid_as_id?(*s)
+    s.all? { |q|
+      q = q.to_s
+      !q.to_s.empty? && !q.include?("..") && /\A[\w.-]+\z/ =~ q
+    }
+  end
+
+  def normalize_str(s)
+    s.gsub("/", "_").gsub("..", "__")
+  end
+
+  def equal_str?(s, t)
+    sb = s.bytes; tb = t.bytes
+    sb.size == tb.size && sb.each_with_index.map { |b, i| tb[i] == b }.all?
+  end
+
+  def find_jobs_for_project(project)
+    Dir[File.join(NyaConfig.datadir, project, "*")]
+      .select { |d| File.directory?(d) }
+      .map { |d| File.basename(d) }
+      .sort.reverse
+  end
+
+  def find_logs_for_job(project, job)
+    Dir[File.join(NyaConfig.datadir, project, job, "*")]
+      .select { |f| File.file?(f) }
+      .map { |f| File.basename(f).sub(/\.log\z/, "") }
+      .sort
+  end
+
+  def open_log(project, job, name)
+    File.open(File.join(NyaConfig.datadir, project, job, name + ".log")) { |f|
+      yield f
+    }
+  end
+
+  def read_log_meta(project, job, name)
+    open_log(project, job, name) { |f|
+      # FIXME
+      lines = f.readlines
+      result_line = lines.reverse_each.find { |l| /\ARESULT=/ =~ l }
+      result = result_line&.split("=")&.last&.chomp
+      time_line = lines.reverse_each.find { |l| /\ATIME=/ =~ l }
+      time = time_line&.split("=")&.last&.chomp
+
+      { result: result, time: time }
+    }
+  end
+
+  def flat_time
+    Time.now.utc.iso8601.delete("-:")
+  end
+end
diff --git a/webapp.rb b/webapp.rb
new file mode 100644
index 0000000..7fae1c6
--- /dev/null
+++ b/webapp.rb
@@ -0,0 +1,125 @@
+require "sinatra"
+require_relative "nyacommon"
+require_relative "config"
+
+include ERB::Util
+
+Thread.report_on_exception = true
+$queue = Queue.new
+Thread.new {
+  while item = $queue.pop
+    system("ruby", File.expand_path("../nyabuild.rb", __FILE__),
+           item[0], item[1])
+  end
+}
+
+get "/" do
+  @projects = NyaConfig.projects.keys
+  erb :index
+end
+
+get "/robots.txt" do
+  <<~END
+    User-agent: *
+    Disallow: /
+  END
+end
+
+get "/:project" do |project|
+  halt 400 unless NyaUtils.valid_as_id?(project)
+
+  @jobs = NyaUtils.find_jobs_for_project(project).take(20).map { |jobid|
+    pats = NyaUtils.find_logs_for_job(project, jobid).map { |name|
+      meta = NyaUtils.read_log_meta(project, jobid, name)
+
+      [name, meta]
+    }
+
+    [jobid, pats]
+  }
+  erb :project
+end
+
+get "/:project/:jobid" do |project, jobid|
+  halt 400 unless NyaUtils.valid_as_id?(project, jobid)
+
+  @patterns = NyaUtils.find_logs_for_job(project, jobid).map { |name|
+    meta = NyaUtils.read_log_meta(project, jobid, name)
+    [name, meta]
+  }
+
+  erb :job
+end
+
+get "/:project/:jobid/:pattern" do |project, jobid, pattern|
+  halt 400 unless NyaUtils.valid_as_id?(project, jobid, pattern)
+
+  @logid = "#{project}/#{jobid}/#{pattern}"
+  @log = NyaUtils.open_log(project, jobid, pattern, &:read)
+
+  erb :log
+end
+
+post "/:project" do |project|
+  halt 401 unless NyaUtils.equal_str?(NyaConfig.secret, params[:secret])
+  halt 400 unless NyaUtils.valid_as_id?(project) && params[:branch]
+  halt 404 unless NyaConfig.project(project.intern)
+
+  $queue << [project, params[:branch]]
+  "queued"
+end
+
+__END__
+@@ layout
+<!DOCTYPE html>
+<meta charset=UTF-8>
+<meta name=viewport content="width=device-width,initial-scale=1">
+<title>ci.rhe.jp</title>
+<style>
+.unknown { color: gray; }
+.success { color: green; }
+.failure { color: red; }
+</style>
+<!--
+   ∧,,,∧
+  (     )
+-->
+<%= yield %>
+
+@@ index
+<h1>Projects</h1>
+<ul>
+  <% @projects.each do |proj| %>
+  <li><a href="/<%=h proj %>"><%=h proj %></a>
+  <% end %>
+</ul>
+
+@@ project
+<h1><%=h params[:project] %></h1>
+<% @jobs.each do |jobid, pats| %>
+<h2><%=h jobid %></h2>
+<ul>
+<% pats.each do |pat, meta| %>
+<li><span class="<%=h meta[:result]&.chomp || "unknown" %>">
+      <%=h meta[:result]&.chomp || "unknown" %></span>
+    <span><%=h meta[:time] ? "%.2fs" % meta[:time].to_f : "-" %></span>
+    <a href="/<%=h params[:project] %>/<%=h jobid %>/<%=h pat %>"><%=h pat %></a>
+<% end %>
+</ul>
+<% end %>
+
+@@ job
+<h1><%=h params[:project] %>/<%=h params[:jobid] %></h1>
+<ul>
+<% @patterns.each do |pat, meta| %>
+<li><span class="<%=h meta[:result]&.chomp || "unknown" %>">
+      <%=h meta[:result]&.chomp || "unknown" %></span>
+    <span><%=h meta[:time] ? "%.2fs" % meta[:time].to_f : "-" %></span>
+    <a href="/<%=h params[:project] %>/<%=h params[:jobid] %>/<%=h pat %>"
+      ><%=h pat %></a>
+<% end %>
+</ul>
+
+@@ log
+<h1><%=h @logid %></h1>
+<pre><%=h @log %></pre>