1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
|
require "fileutils"
require "pathname"
require "shellwords"
require "time"
require "open3"
require "tmpdir"
require_relative "nyacommon"
require_relative "config"
class NyaLogger
attr_reader :io
def initialize(path)
@io = File.open(path, "w")
@tag_map = {}
@begin_time = Time.now
puts "= begin # #{NyaUtils.flat_time}"
end
def item(tag)
if @tag_map[tag]
tag += "(#{@tag_map[tag] += 1})"
else
@tag_map[tag] = 1
end
puts "== #{tag} # #{NyaUtils.flat_time}"
yield
puts
end
def log(str)
puts "+ #{str}"
end
def warn(str)
puts "++ #{str}"
end
def finish(success)
puts "= end # #{NyaUtils.flat_time}"
puts "RESULT=#{success ? "success" : "failure"}"
puts "TIME=#{Time.now - @begin_time}"
@io.close
end
private def puts(str = "")
@io.puts(str)
@io.flush
end
end
class NyaBuildTarget
def initialize(build, path, logger)
@build = build
@path = path
@tag_map = {}
@logger = logger
end
private def run0(cmd, timeout: nil)
@logger.log(cmd)
pid = spawn({}, *cmd, pgroup: true, [:out, :err] => @logger.io)
th = Thread.new { _, status = Process.waitpid2(-pid); status }
unless th.join(timeout)
th.kill; th.join
@logger.warn("Execution timed out (%p sec)", timeout)
Process.kill(:KILL, -pid)
_, status = Process.waitpid2(-pid)
raise status.inspect
end
unless (status = th.value).success?
@logger.warn(status.inspect)
raise status.inspect
end
end
def run(cmd, **opts)
@logger.item(cmd.shellsplit[0]) {
run0(cmd, **opts)
}
end
def project() @build.project end
def cdir
Pathname.getwd.relative_path_from(Pathname.new(@path)).to_s
end
def s(str)
Shellwords.shellescape(str)
end
def git(repo, dest, branch:)
@logger.item("git/clone") {
run0 "git clone --depth 1 -q -b #{s branch} #{s repo} #{s dest}"
run0 "git -C #{s dest} log --max-count=1"
}
end
def self.start(build, confs, logger)
Dir.mktmpdir(nil, NyaConfig.tmpdir) { |path|
FileUtils.cd(path) {
t = new(build, path, logger)
t.git(t.project.git, build.project_id, branch: build.branch)
FileUtils.cd(build.project_id) {
t.project.build_proc.call(t, **confs)
}
}
}
rescue Exception
logger.warn("Exception: %s (%s)", $?.class, $?.message)
raise
end
end
class NyaBuild
attr_reader :project, :project_id, :branch
def initialize(project_id, branch, jobid = "#{NyaUtils.flat_time}-#{branch}")
@project = NyaConfig.project(project_id.intern)
@project_id = project_id
@branch = branch
@jobid = NyaUtils.normalize_str(jobid)
@jobdir = File.join(NyaConfig.datadir, @project_id, @jobid)
FileUtils.mkdir_p(@jobdir)
@results = []
end
def run
matrix = @project.matrix.reverse
beg = Time.now
NyaConfig.parallelism.times.map { |x|
Thread.start {
while params = matrix.pop
do_pattern(params)
end
}
}.map(&:join)
elapsed = Time.now - beg
NyaConfig.notification.publish(@project_id, @jobid, @results, elapsed)
end
def do_pattern(params)
pattern_tag = params.values.join("_")
logger = newlogger(pattern_tag)
pid = fork { NyaBuildTarget.start(self, params, logger) }
_, status = Process.wait2 pid
logger.finish(status.success?)
@results << [pattern_tag, status.success?]
end
def newlogger(pattern_tag)
NyaLogger.new(File.join(@jobdir, "#{pattern_tag}.log"))
end
end
if $0 == __FILE__
project_id, branch = ARGV[0], ARGV[1]
unless project_id && branch
warn "usage: nyabuild.rb <project_id> <branch>"
abort
end
NyaBuild.new(project_id, branch).run
end
|
