Class: Metanorma::Compile

Inherits:
Object
  • Object
show all
Defined in:
lib/metanorma/compile/compile.rb,
lib/metanorma/compile/extract.rb,
lib/metanorma/compile/compile_options.rb,
lib/metanorma/compile/compile_validate.rb

Constant Summary collapse

REQUIREMENT_XPATH = 
"//requirement | //xmlns:requirement | //recommendation | "\
"//xmlns:recommendation | //permission | //xmlns:permission".freeze

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize ⇒ Compile

Returns a new instance of Compile.



20
21
22
23
24
25
26
 
# File 'lib/metanorma/compile/compile.rb', line 20

def initialize
  @registry = Metanorma::Registry.instance
  @errors = []
  @isodoc = IsoDoc::Convert.new({})
  @fontist_installed = false
  @log = Metanorma::Utils::Log.new
end

Instance Attribute Details

#errors ⇒ Array<String> (readonly)

Returns:



18
19
20
 
# File 'lib/metanorma/compile/compile.rb', line 18

def errors
  @errors
end

#processor ⇒ Array<String> (readonly)

Returns:



18
19
20
 
# File 'lib/metanorma/compile/compile.rb', line 18

def processor
  @processor
end

Instance Method Details

#clean_exit(options) ⇒ Object 



47
48
49
50
 
# File 'lib/metanorma/compile/compile.rb', line 47

def clean_exit(options)
  options[:novalid] and return
  @log.write
end

#clean_sourcecode(xml) ⇒ Object 



13
14
15
16
17
18
 
# File 'lib/metanorma/compile/extract.rb', line 13

def clean_sourcecode(xml)
  xml.xpath(".//callout | .//annotation | .//xmlns:callout | "\
            ".//xmlns:annotation").each(&:remove)
  xml.xpath(".//br | .//xmlns:br").each { |x| x.replace("\n") }
  HTMLEntities.new.decode(xml.children.to_xml)
end

#compile(filename, options = {}) ⇒ Object 



28
29
30
31
32
33
34
35
36
37
 
# File 'lib/metanorma/compile/compile.rb', line 28

def compile(filename, options = {})
  options_process(filename, options)
  @processor = @registry.find_processor(options[:type].to_sym)
  extensions = get_extensions(options) or return nil
  (file, isodoc = process_input(filename, options)) or return nil
  relaton_export(isodoc, options)
  extract(isodoc, options[:extract], options[:extract_type])
  process_exts(filename, extensions, file, isodoc, options)
  clean_exit(options)
end

#export_output(fname, content, **options) ⇒ Object 



96
97
98
99
 
# File 'lib/metanorma/compile/compile.rb', line 96

def export_output(fname, content, **options)
  mode = options[:binary] ? "wb" : "w:UTF-8"
  File.open(fname, mode) { |f| f.write content }
end

#extract(isodoc, dirname, extract_types) ⇒ Object 



20
21
22
23
24
25
26
27
28
29
30
31
32
 
# File 'lib/metanorma/compile/extract.rb', line 20

def extract(isodoc, dirname, extract_types)
  return unless dirname

  extract_types.nil? || extract_types.empty? and
    extract_types = %i[sourcecode image requirement]
  FileUtils.rm_rf dirname
  FileUtils.mkdir_p dirname
  xml = Nokogiri::XML(isodoc, &:huge)
  sourcecode_export(xml, dirname) if extract_types.include? :sourcecode
  image_export(xml, dirname) if extract_types.include? :image
  extract_types.include?(:requirement) and
    requirement_export(xml, dirname)
end

#extract_extensions(options) ⇒ Object 



47
48
49
50
51
52
53
54
55
56
57
 
# File 'lib/metanorma/compile/compile_options.rb', line 47

def extract_extensions(options)
  options[:extension_keys] ||=
    @processor.output_formats.reduce([]) { |memo, (k, _)| memo << k }
  options[:extension_keys].reduce([]) do |memo, e|
    if @processor.output_formats[e] then memo << e
    else
      unsupported_format_error(e)
      memo
    end
  end
end

#font_install(opt) ⇒ Object 



59
60
61
62
63
 
# File 'lib/metanorma/compile/compile_options.rb', line 59

def font_install(opt)
  @fontist_installed or
    Util::FontistHelper.install_fonts(@processor, opt)
  @fontist_installed = true
end

#gather_and_install_fonts(file, options, extensions) ⇒ Object 



129
130
131
132
133
134
 
# File 'lib/metanorma/compile/compile.rb', line 129

def gather_and_install_fonts(file, options, extensions)
  Util.sort_extensions_execution(extensions).each do |ext|
    isodoc_options = get_isodoc_options(file, options, ext)
    font_install(isodoc_options.merge(options))
  end
end

#get_extensions(options) ⇒ Object 



37
38
39
40
41
42
43
44
45
 
# File 'lib/metanorma/compile/compile_options.rb', line 37

def get_extensions(options)
  ext = extract_extensions(options)
  !ext.include?(:presentation) && ext.any? do |e|
    @processor.use_presentation_xml(e)
  end and ext << :presentation
  !ext.include?(:rxl) && options[:site_generate] and
    ext << :rxl
  ext
end

#image_export(xml, dirname) ⇒ Object 



44
45
46
47
48
49
50
51
52
53
54
55
56
 
# File 'lib/metanorma/compile/extract.rb', line 44

def image_export(xml, dirname)
  xml.at("//image | //xmlns:image") or return
  FileUtils.mkdir_p "#{dirname}/image"
  xml.xpath("//image | //xmlns:image").each_with_index do |s, i|
    next unless /^data:image/.match? s["src"]

    %r{^data:image/(?<imgtype>[^;]+);base64,(?<imgdata>.+)$} =~ s["src"]
    fn = s["filename"] || sprintf("image-%<num>04d.%<name>s",
                                  num: i, name: imgtype)
    export_output("#{dirname}/image/#{fn}", Base64.strict_decode64(imgdata),
                  binary: true)
  end
end

#load_flavor(stdtype) ⇒ Object 



24
25
26
27
28
29
30
31
32
33
34
 
# File 'lib/metanorma/compile/compile_validate.rb', line 24

def load_flavor(stdtype)
  stdtype = stdtype.to_sym
  flavor = stdtype2flavor(stdtype)
  @registry.supported_backends.include? stdtype or
    Util.log("[metanorma] Info: Loading `#{flavor}` gem "\
             "for standard type `#{stdtype}`.", :info)
  require_flavor(flavor)
  @registry.supported_backends.include? stdtype or
    Util.log("[metanorma] Error: The `#{flavor}` gem does not "\
             "support the standard type #{stdtype}. Exiting.", :fatal)
end

#options_extract(filename, options) ⇒ Object 



19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
 
# File 'lib/metanorma/compile/compile_options.rb', line 19

def options_extract(filename, options)
  content = read_file(filename)
  o = Metanorma::Input::Asciidoc.new.extract_metanorma_options(content)
    .merge(xml_options_extract(content))
  options[:type] ||= o[:type]&.to_sym
  t = @registry.alias(options[:type]) and options[:type] = t
  dir = filename.sub(%r(/[^/]+$), "/")
  options[:relaton] ||= File.join(dir, o[:relaton]) if o[:relaton]
  options[:sourcecode] ||= File.join(dir, o[:sourcecode]) if o[:sourcecode]
  options[:extension_keys] ||= o[:extensions]&.split(/, */)&.map(&:to_sym)
  options[:extension_keys] = nil if options[:extension_keys] == [:all]
  options[:format] ||= :asciidoc
  options[:filename] = filename
  options[:fontlicenseagreement] ||= "no-install-fonts"
  options[:novalid] = o[:novalid] if o[:novalid]
  options
end

#options_process(filename, options) ⇒ Object 



39
40
41
42
43
44
45
 
# File 'lib/metanorma/compile/compile.rb', line 39

def options_process(filename, options)
  require_libraries(options)
  options = options_extract(filename, options)
  validate_options(options)
  @log.save_to(filename, options[:output_dir])
  options[:log] = @log
end

#process_ext(ext, file, isodoc, fnames, options) ⇒ Object 



136
137
138
139
140
141
142
143
144
145
146
 
# File 'lib/metanorma/compile/compile.rb', line 136

def process_ext(ext, file, isodoc, fnames, options)
  fnames[:ext] = @processor.output_formats[ext]
  fnames[:out] = fnames[:f].sub(/\.[^.]+$/, ".#{fnames[:ext]}")
  isodoc_options = get_isodoc_options(file, options, ext)
  thread = true
  unless process_ext_simple(ext, isodoc, fnames, options,
                            isodoc_options)
    thread = process_exts1(ext, fnames, isodoc, options, isodoc_options)
  end
  thread
end

#process_ext_simple(ext, isodoc, fnames, options, isodoc_options) ⇒ Object 



148
149
150
151
152
153
154
155
156
157
158
159
160
161
 
# File 'lib/metanorma/compile/compile.rb', line 148

def process_ext_simple(ext, isodoc, fnames, options, isodoc_options)
  if ext == :rxl
    relaton_export(isodoc, options.merge(relaton: fnames[:out]))
  elsif options[:passthrough_presentation_xml] && ext == :presentation
    #f = File.exist?(fnames[:f]) ? fnames[:f] : fnames[:orig_filename]
    f = File.exist?(fnames[:orig_filename]) ? fnames[:orig_filename] : fnames[:f]
    FileUtils.cp f, fnames[:presentationxml]
  elsif ext == :html && options[:sectionsplit]
    sectionsplit_convert(fnames[:xml], isodoc, fnames[:out],
                         isodoc_options)
  else return false
  end
  true
end

#process_exts(filename, extensions, file, isodoc, options) ⇒ Object

isodoc is Raw Metanorma XML



111
112
113
114
115
116
117
118
119
120
121
 
# File 'lib/metanorma/compile/compile.rb', line 111

def process_exts(filename, extensions, file, isodoc, options)
  f = File.expand_path(change_output_dir(options))
  fnames = { xml: f.sub(/\.[^.]+$/, ".xml"), f: f,
             orig_filename: File.expand_path(filename),
             presentationxml: f.sub(/\.[^.]+$/, ".presentation.xml") }
  @queue = ::Metanorma::Util::WorkersPool
    .new(ENV["METANORMA_PARALLEL"]&.to_i || 3)
  gather_and_install_fonts(file, options.dup, extensions)
  process_exts_run(fnames, file, isodoc, extensions, options)
  @queue.shutdown
end

#process_exts1(ext, fnames, isodoc, options, isodoc_options) ⇒ Object 



163
164
165
166
167
168
169
170
171
172
 
# File 'lib/metanorma/compile/compile.rb', line 163

def process_exts1(ext, fnames, isodoc, options, isodoc_options)
  if @processor.use_presentation_xml(ext)
    @queue.schedule(ext, fnames.dup, options.dup,
                    isodoc_options.dup) do |a, b, c, d|
      process_output_threaded(a, b, c, d)
    end
  else
    process_output_unthreaded(ext, fnames, isodoc, isodoc_options)
  end
end

#process_exts_run(fnames, file, isodoc, extensions, options) ⇒ Object 



123
124
125
126
127
 
# File 'lib/metanorma/compile/compile.rb', line 123

def process_exts_run(fnames, file, isodoc, extensions, options)
  Util.sort_extensions_execution(extensions).each do |ext|
    process_ext(ext, file, isodoc, fnames, options) or break
  end
end

#process_input(filename, options) ⇒ Object 



52
53
54
55
56
57
58
59
60
61
 
# File 'lib/metanorma/compile/compile.rb', line 52

def process_input(filename, options)
  case extname = File.extname(filename)
  when ".adoc" then process_input_adoc(filename, options)
  when ".xml" then process_input_xml(filename, options)
  else
    Util.log("[metanorma] Error: file extension #{extname} " \
             "is not supported.", :error)
    nil
  end
end

#process_input_adoc(filename, options) ⇒ Object 



63
64
65
66
67
68
69
70
71
72
73
 
# File 'lib/metanorma/compile/compile.rb', line 63

def process_input_adoc(filename, options)
  Util.log("[metanorma] Processing: AsciiDoc input.", :info)
  file = read_file(filename)
  options[:asciimath] and
    file.sub!(/^(=[^\n]+\n)/, "\\1:mn-keep-asciimath:\n")
  dir = File.dirname(filename)
  dir != "." and
    file = file.gsub(/^include::/, "include::#{dir}/")
      .gsub(/^embed::/, "embed::#{dir}/")
  [file, @processor.input_to_isodoc(file, filename, options)]
end

#process_input_xml(filename, _options) ⇒ Object 



75
76
77
78
79
80
 
# File 'lib/metanorma/compile/compile.rb', line 75

def process_input_xml(filename, _options)
  Util.log("[metanorma] Processing: Metanorma XML input.", :info)
  # TODO NN: this is a hack -- we should provide/bridge the
  # document attributes in Metanorma XML
  ["", read_file(filename)]
end

#process_output_threaded(ext, fnames1, options1, isodoc_options1) ⇒ Object 



174
175
176
177
178
179
180
181
 
# File 'lib/metanorma/compile/compile.rb', line 174

def process_output_threaded(ext, fnames1, options1, isodoc_options1)
  @processor.output(nil, fnames1[:presentationxml], fnames1[:out], ext,
                    isodoc_options1)
  wrap_html(options1, fnames1[:ext], fnames1[:out])
rescue StandardError => e
  strict = ext == :presentation || isodoc_options1[:strict] == true
  isodoc_error_process(e, strict, false)
end

#process_output_unthreaded(ext, fnames, isodoc, isodoc_options) ⇒ Object 



183
184
185
186
187
188
189
190
191
 
# File 'lib/metanorma/compile/compile.rb', line 183

def process_output_unthreaded(ext, fnames, isodoc, isodoc_options)
  @processor.output(isodoc, fnames[:xml], fnames[:out], ext,
                    isodoc_options)
  true # return as Thread
rescue StandardError => e
  strict = ext == :presentation || isodoc_options[:strict] == "true"
  isodoc_error_process(e, strict, true)
  ext != :presentation
end

#read_file(filename) ⇒ Object 



82
83
84
 
# File 'lib/metanorma/compile/compile.rb', line 82

def read_file(filename)
  File.read(filename, encoding: "utf-8").gsub("\r\n", "\n")
end

#relaton_export(isodoc, options) ⇒ Object 



86
87
88
89
90
91
92
93
94
 
# File 'lib/metanorma/compile/compile.rb', line 86

def relaton_export(isodoc, options)
  return unless options[:relaton]

  xml = Nokogiri::XML(isodoc, &:huge)
  bibdata = xml.at("//bibdata") || xml.at("//xmlns:bibdata")
  # docid = bibdata&.at("./xmlns:docidentifier")&.text || options[:filename]
  # outname = docid.sub(/^\s+/, "").sub(/\s+$/, "").gsub(/\s+/, "-") + ".xml"
  File.open(options[:relaton], "w:UTF-8") { |f| f.write bibdata.to_xml }
end

#require_libraries(options) ⇒ Object 



5
6
7
 
# File 'lib/metanorma/compile/compile_options.rb', line 5

def require_libraries(options)
  options&.dig(:require)&.each { |r| require r }
end

#requirement_export(xml, dirname) ⇒ Object 



62
63
64
65
66
67
68
69
70
 
# File 'lib/metanorma/compile/extract.rb', line 62

def requirement_export(xml, dirname)
  xml.at(REQUIREMENT_XPATH) or return
  FileUtils.mkdir_p "#{dirname}/requirement"
  xml.xpath(REQUIREMENT_XPATH).each_with_index do |s, i|
    fn = s["filename"] ||
      sprintf("%<name>s-%<num>04d.xml", name: s.name, num: i)
    export_output("#{dirname}/requirement/#{fn}", s)
  end
end

#sectionsplit_convert(input_filename, file, output_filename = nil, opts = {}) ⇒ Object

assume we pass in Presentation XML, but we want to recover Semantic XML



194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
 
# File 'lib/metanorma/compile/compile.rb', line 194

def sectionsplit_convert(input_filename, file, output_filename = nil,
                         opts = {})
  @isodoc ||= IsoDoc::PresentationXMLConvert.new({})
  input_filename += ".xml" unless input_filename.match?(/\.xml$/)
  File.exist?(input_filename) or
    File.open(input_filename, "w:UTF-8") { |f| f.write(file) }
  presxml = File.read(input_filename, encoding: "utf-8")
  _xml, filename, dir = @isodoc.convert_init(presxml, input_filename, false)

  ::Metanorma::Collection::Sectionsplit.new(
    input: input_filename,
    isodoc: @isodoc,
    xml: presxml,
    base: File.basename(output_filename || filename),
    output: output_filename || filename,
    dir: dir,
    compile_opts: opts,
  ).build_collection
end

#sourcecode_export(xml, dirname) ⇒ Object 



34
35
36
37
38
39
40
41
42
 
# File 'lib/metanorma/compile/extract.rb', line 34

def sourcecode_export(xml, dirname)
  xml.at("//sourcecode | //xmlns:sourcecode") or return
  FileUtils.mkdir_p "#{dirname}/sourcecode"
  xml.xpath("//sourcecode | //xmlns:sourcecode").each_with_index do |s, i|
    filename = s["filename"] || sprintf("sourcecode-%04d.txt", i)
    export_output("#{dirname}/sourcecode/#{filename}",
                  clean_sourcecode(s.dup))
  end
end

#validate_format(options) ⇒ Object 



17
18
19
20
21
22
 
# File 'lib/metanorma/compile/compile_validate.rb', line 17

def validate_format(options)
  unless options[:format] == :asciidoc
    Util.log("[metanorma] Error: Only source file format currently "\
             "supported is 'asciidoc'.", :fatal)
  end
end

#validate_options(options) ⇒ Object 



3
4
5
6
 
# File 'lib/metanorma/compile/compile_validate.rb', line 3

def validate_options(options)
  validate_type(options)
  validate_format(options)
end

#validate_type(options) ⇒ Object 



8
9
10
11
12
13
14
15
 
# File 'lib/metanorma/compile/compile_validate.rb', line 8

def validate_type(options)
  unless options[:type]
    Util.log("[metanorma] Error: Please specify a standard type: "\
             "#{@registry.supported_backends}.", :fatal)
  end
  stdtype = options[:type].to_sym
  load_flavor(stdtype)
end

#wrap_html(options, file_extension, outfilename) ⇒ Object 



101
102
103
104
105
106
107
108
 
# File 'lib/metanorma/compile/compile.rb', line 101

def wrap_html(options, file_extension, outfilename)
  if options[:wrapper] && /html$/.match(file_extension)
    outfilename = outfilename.sub(/\.html$/, "")
    FileUtils.mkdir_p outfilename
    FileUtils.mv "#{outfilename}.html", outfilename
    FileUtils.mv "#{outfilename}_images", outfilename, force: true
  end
end

#xml_options_extract(file) ⇒ Object 



9
10
11
12
13
14
15
16
17
 
# File 'lib/metanorma/compile/compile_options.rb', line 9

def xml_options_extract(file)
  xml = Nokogiri::XML(file, &:huge)
  if xml.root
    @registry.root_tags.each do |k, v|
      return { type: k } if v == xml.root.name
    end
  end
  {}
end