Class: Metanorma::Compile

Inherits:
Object
  • Object
show all
Defined in:
lib/metanorma/compile.rb,
lib/metanorma/extract.rb,
lib/metanorma/compile_options.rb,
lib/metanorma/compile_validate.rb

Constant Summary collapse

REQUIREMENT_XPATH = 
"//requirement | //xmlns:requirement | //recommendation | "\
"//xmlns:recommendation | //permission | //xmlns:permission".freeze

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize ⇒ Compile

Returns a new instance of Compile.



20
21
22
23
24
25
26
 
# File 'lib/metanorma/compile.rb', line 20

def initialize
  @registry = Metanorma::Registry.instance
  @errors = []
  @isodoc = IsoDoc::Convert.new({})
  @fontist_installed = false
  @log = Metanorma::Utils::Log.new
end

Instance Attribute Details

#errors ⇒ Array<String> (readonly)

Returns:

  • (Array<String>) 


18
19
20
 
# File 'lib/metanorma/compile.rb', line 18

def errors
  @errors
end

#processor ⇒ Array<String> (readonly)

Returns:

  • (Array<String>) 


18
19
20
 
# File 'lib/metanorma/compile.rb', line 18

def processor
  @processor
end

Instance Method Details

#clean_exit(options) ⇒ Object 



47
48
49
50
 
# File 'lib/metanorma/compile.rb', line 47

def clean_exit(options)
  options[:novalid] and return
  @log.write
end

#clean_sourcecode(xml) ⇒ Object 



13
14
15
16
17
18
 
# File 'lib/metanorma/extract.rb', line 13

def clean_sourcecode(xml)
  xml.xpath(".//callout | .//annotation | .//xmlns:callout | "\
            ".//xmlns:annotation").each(&:remove)
  xml.xpath(".//br | .//xmlns:br").each { |x| x.replace("\n") }
  HTMLEntities.new.decode(xml.children.to_xml)
end

#compile(filename, options = {}) ⇒ Object 



28
29
30
31
32
33
34
35
36
37
 
# File 'lib/metanorma/compile.rb', line 28

def compile(filename, options = {})
  options_process(filename, options)
  @processor = @registry.find_processor(options[:type].to_sym)
  extensions = get_extensions(options) or return nil
  (file, isodoc = process_input(filename, options)) or return nil
  relaton_export(isodoc, options)
  extract(isodoc, options[:extract], options[:extract_type])
  process_exts(filename, extensions, file, isodoc, options)
  clean_exit(options)
end

#export_output(fname, content, **options) ⇒ Object 



96
97
98
99
 
# File 'lib/metanorma/compile.rb', line 96

def export_output(fname, content, **options)
  mode = options[:binary] ? "wb" : "w:UTF-8"
  File.open(fname, mode) { |f| f.write content }
end

#extract(isodoc, dirname, extract_types) ⇒ Object 



20
21
22
23
24
25
26
27
28
29
30
31
32
 
# File 'lib/metanorma/extract.rb', line 20

def extract(isodoc, dirname, extract_types)
  return unless dirname

  extract_types.nil? || extract_types.empty? and
    extract_types = %i[sourcecode image requirement]
  FileUtils.rm_rf dirname
  FileUtils.mkdir_p dirname
  xml = Nokogiri::XML(isodoc) { |config| config.huge }
  sourcecode_export(xml, dirname) if extract_types.include? :sourcecode
  image_export(xml, dirname) if extract_types.include? :image
  extract_types.include?(:requirement) and
    requirement_export(xml, dirname)
end

#extract_extensions(options) ⇒ Object 



47
48
49
50
51
52
53
54
55
56
57
 
# File 'lib/metanorma/compile_options.rb', line 47

def extract_extensions(options)
  options[:extension_keys] ||=
    @processor.output_formats.reduce([]) { |memo, (k, _)| memo << k }
  options[:extension_keys].reduce([]) do |memo, e|
    if @processor.output_formats[e] then memo << e
    else
      unsupported_format_error(e)
      memo
    end
  end
end

#font_install(opt) ⇒ Object 



59
60
61
62
 
# File 'lib/metanorma/compile_options.rb', line 59

def font_install(opt)
  FontistUtils.install_fonts(@processor, opt) unless @fontist_installed
  @fontist_installed = true
end

#gather_and_install_fonts(file, options, extensions) ⇒ Object 



129
130
131
132
133
134
 
# File 'lib/metanorma/compile.rb', line 129

def gather_and_install_fonts(file, options, extensions)
  Util.sort_extensions_execution(extensions).each do |ext|
    isodoc_options = get_isodoc_options(file, options, ext)
    font_install(isodoc_options.merge(options))
  end
end

#get_extensions(options) ⇒ Object 



37
38
39
40
41
42
43
44
45
 
# File 'lib/metanorma/compile_options.rb', line 37

def get_extensions(options)
  ext = extract_extensions(options)
  !ext.include?(:presentation) && ext.any? do |e|
    @processor.use_presentation_xml(e)
  end and ext << :presentation
  !ext.include?(:rxl) && options[:site_generate] and
    ext << :rxl
  ext
end

#image_export(xml, dirname) ⇒ Object 



44
45
46
47
48
49
50
51
52
53
54
55
56
 
# File 'lib/metanorma/extract.rb', line 44

def image_export(xml, dirname)
  xml.at("//image | //xmlns:image") or return
  FileUtils.mkdir_p "#{dirname}/image"
  xml.xpath("//image | //xmlns:image").each_with_index do |s, i|
    next unless /^data:image/.match? s["src"]

    %r{^data:image/(?<imgtype>[^;]+);base64,(?<imgdata>.+)$} =~ s["src"]
    fn = s["filename"] || sprintf("image-%<num>04d.%<name>s",
                                  num: i, name: imgtype)
    export_output("#{dirname}/image/#{fn}", Base64.strict_decode64(imgdata),
                  binary: true)
  end
end

#options_extract(filename, options) ⇒ Object 



19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
 
# File 'lib/metanorma/compile_options.rb', line 19

def options_extract(filename, options)
  content = read_file(filename)
  o = Metanorma::Input::Asciidoc.new.extract_metanorma_options(content)
    .merge(xml_options_extract(content))
  options[:type] ||= o[:type]&.to_sym
  t = @registry.alias(options[:type]) and options[:type] = t
  dir = filename.sub(%r(/[^/]+$), "/")
  options[:relaton] ||= File.join(dir, o[:relaton]) if o[:relaton]
  options[:sourcecode] ||= File.join(dir, o[:sourcecode]) if o[:sourcecode]
  options[:extension_keys] ||= o[:extensions]&.split(/, */)&.map(&:to_sym)
  options[:extension_keys] = nil if options[:extension_keys] == [:all]
  options[:format] ||= :asciidoc
  options[:filename] = filename
  options[:fontlicenseagreement] ||= "no-install-fonts"
  options[:novalid] = o[:novalid] if o[:novalid]
  options
end

#options_process(filename, options) ⇒ Object 



39
40
41
42
43
44
45
 
# File 'lib/metanorma/compile.rb', line 39

def options_process(filename, options)
  require_libraries(options)
  options = options_extract(filename, options)
  validate_options(options)
  @log.save_to(filename, options[:output_dir])
  options[:log] = @log
end

#process_ext(ext, file, isodoc, fnames, options) ⇒ Object 



136
137
138
139
140
141
142
143
144
145
146
 
# File 'lib/metanorma/compile.rb', line 136

def process_ext(ext, file, isodoc, fnames, options)
  fnames[:ext] = @processor.output_formats[ext]
  fnames[:out] = fnames[:f].sub(/\.[^.]+$/, ".#{fnames[:ext]}")
  isodoc_options = get_isodoc_options(file, options, ext)
  thread = true
  unless process_ext_simple(ext, isodoc, fnames, options,
                            isodoc_options)
    thread = process_exts1(ext, fnames, isodoc, options, isodoc_options)
  end
  thread
end

#process_ext_simple(ext, isodoc, fnames, options, isodoc_options) ⇒ Object 



148
149
150
151
152
153
154
155
156
157
158
159
160
 
# File 'lib/metanorma/compile.rb', line 148

def process_ext_simple(ext, isodoc, fnames, options, isodoc_options)
  if ext == :rxl
    relaton_export(isodoc, options.merge(relaton: fnames[:out]))
  elsif options[:passthrough_presentation_xml] && ext == :presentation
    f = File.exist?(fnames[:f]) ? fnames[:f] : fnames[:orig_filename]
    FileUtils.cp f, fnames[:presentationxml]
  elsif ext == :html && options[:sectionsplit]
    sectionsplit_convert(fnames[:xml], isodoc, fnames[:out],
                         isodoc_options)
  else return false
  end
  true
end

#process_exts(filename, extensions, file, isodoc, options) ⇒ Object

isodoc is Raw Metanorma XML



111
112
113
114
115
116
117
118
119
120
121
 
# File 'lib/metanorma/compile.rb', line 111

def process_exts(filename, extensions, file, isodoc, options)
  f = File.expand_path(change_output_dir(options))
  fnames = { xml: f.sub(/\.[^.]+$/, ".xml"), f: f,
             orig_filename: File.expand_path(filename),
             presentationxml: f.sub(/\.[^.]+$/, ".presentation.xml") }
  @queue = ::Metanorma::WorkersPool
    .new(ENV["METANORMA_PARALLEL"]&.to_i || 3)
  gather_and_install_fonts(file, options.dup, extensions)
  process_exts_run(fnames, file, isodoc, extensions, options)
  @queue.shutdown
end

#process_exts1(ext, fnames, isodoc, options, isodoc_options) ⇒ Object 



162
163
164
165
166
167
168
169
170
171
 
# File 'lib/metanorma/compile.rb', line 162

def process_exts1(ext, fnames, isodoc, options, isodoc_options)
  if @processor.use_presentation_xml(ext)
    @queue.schedule(ext, fnames.dup, options.dup,
                    isodoc_options.dup) do |a, b, c, d|
      process_output_threaded(a, b, c, d)
    end
  else
    process_output_unthreaded(ext, fnames, isodoc, isodoc_options)
  end
end

#process_exts_run(fnames, file, isodoc, extensions, options) ⇒ Object 



123
124
125
126
127
 
# File 'lib/metanorma/compile.rb', line 123

def process_exts_run(fnames, file, isodoc, extensions, options)
  Util.sort_extensions_execution(extensions).each do |ext|
    process_ext(ext, file, isodoc, fnames, options) or break
  end
end

#process_input(filename, options) ⇒ Object 



52
53
54
55
56
57
58
59
60
61
 
# File 'lib/metanorma/compile.rb', line 52

def process_input(filename, options)
  case extname = File.extname(filename)
  when ".adoc" then process_input_adoc(filename, options)
  when ".xml" then process_input_xml(filename, options)
  else
    Util.log("[metanorma] Error: file extension #{extname} " \
             "is not supported.", :error)
    nil
  end
end

#process_input_adoc(filename, options) ⇒ Object 



63
64
65
66
67
68
69
70
71
72
73
 
# File 'lib/metanorma/compile.rb', line 63

def process_input_adoc(filename, options)
  Util.log("[metanorma] Processing: AsciiDoc input.", :info)
  file = read_file(filename)
  options[:asciimath] and
    file.sub!(/^(=[^\n]+\n)/, "\\1:mn-keep-asciimath:\n")
  dir = File.dirname(filename)
  dir != "." and
    file = file.gsub(/^include::/, "include::#{dir}/")
      .gsub(/^embed::/, "embed::#{dir}/")
  [file, @processor.input_to_isodoc(file, filename, options)]
end

#process_input_xml(filename, _options) ⇒ Object 



75
76
77
78
79
80
 
# File 'lib/metanorma/compile.rb', line 75

def process_input_xml(filename, _options)
  Util.log("[metanorma] Processing: Metanorma XML input.", :info)
  # TODO NN: this is a hack -- we should provide/bridge the
  # document attributes in Metanorma XML
  ["", read_file(filename)]
end

#process_output_threaded(ext, fnames1, options1, isodoc_options1) ⇒ Object 



173
174
175
176
177
178
179
180
 
# File 'lib/metanorma/compile.rb', line 173

def process_output_threaded(ext, fnames1, options1, isodoc_options1)
  @processor.output(nil, fnames1[:presentationxml], fnames1[:out], ext,
                    isodoc_options1)
  wrap_html(options1, fnames1[:ext], fnames1[:out])
rescue StandardError => e
  strict = ext == :presentation || isodoc_options1[:strict] == true
  isodoc_error_process(e, strict, false)
end

#process_output_unthreaded(ext, fnames, isodoc, isodoc_options) ⇒ Object 



182
183
184
185
186
187
188
189
190
 
# File 'lib/metanorma/compile.rb', line 182

def process_output_unthreaded(ext, fnames, isodoc, isodoc_options)
  @processor.output(isodoc, fnames[:xml], fnames[:out], ext,
                    isodoc_options)
  true # return as Thread
rescue StandardError => e
  strict = ext == :presentation || isodoc_options[:strict] == "true"
  isodoc_error_process(e, strict, true)
  ext != :presentation
end

#read_file(filename) ⇒ Object 



82
83
84
 
# File 'lib/metanorma/compile.rb', line 82

def read_file(filename)
  File.read(filename, encoding: "utf-8").gsub("\r\n", "\n")
end

#relaton_export(isodoc, options) ⇒ Object 



86
87
88
89
90
91
92
93
94
 
# File 'lib/metanorma/compile.rb', line 86

def relaton_export(isodoc, options)
  return unless options[:relaton]

  xml = Nokogiri::XML(isodoc) { |config| config.huge }
  bibdata = xml.at("//bibdata") || xml.at("//xmlns:bibdata")
  # docid = bibdata&.at("./xmlns:docidentifier")&.text || options[:filename]
  # outname = docid.sub(/^\s+/, "").sub(/\s+$/, "").gsub(/\s+/, "-") + ".xml"
  File.open(options[:relaton], "w:UTF-8") { |f| f.write bibdata.to_xml }
end

#require_libraries(options) ⇒ Object 



5
6
7
 
# File 'lib/metanorma/compile_options.rb', line 5

def require_libraries(options)
  options&.dig(:require)&.each { |r| require r }
end

#requirement_export(xml, dirname) ⇒ Object 



62
63
64
65
66
67
68
69
70
 
# File 'lib/metanorma/extract.rb', line 62

def requirement_export(xml, dirname)
  xml.at(REQUIREMENT_XPATH) or return
  FileUtils.mkdir_p "#{dirname}/requirement"
  xml.xpath(REQUIREMENT_XPATH).each_with_index do |s, i|
    fn = s["filename"] ||
      sprintf("%<name>s-%<num>04d.xml", name: s.name, num: i)
    export_output("#{dirname}/requirement/#{fn}", s)
  end
end

#sectionsplit_convert(input_filename, file, output_filename = nil, opts = {}) ⇒ Object

assume we pass in Presentation XML, but we want to recover Semantic XML



193
194
195
196
197
198
199
200
201
202
203
204
205
 
# File 'lib/metanorma/compile.rb', line 193

def sectionsplit_convert(input_filename, file, output_filename = nil,
                         opts = {})
  @isodoc ||= IsoDoc::PresentationXMLConvert.new({})
  input_filename += ".xml" unless input_filename.match?(/\.xml$/)
  File.exist?(input_filename) or
    File.open(input_filename, "w:UTF-8") { |f| f.write(file) }
  presxml = File.read(input_filename, encoding: "utf-8")
  _xml, filename, dir = @isodoc.convert_init(presxml, input_filename, false)
  Sectionsplit.new(input: input_filename, isodoc: @isodoc, xml: presxml,
                   base: File.basename(output_filename || filename),
                   output: output_filename || filename,
                   dir: dir, compile_opts: opts).build_collection
end

#sourcecode_export(xml, dirname) ⇒ Object 



34
35
36
37
38
39
40
41
42
 
# File 'lib/metanorma/extract.rb', line 34

def sourcecode_export(xml, dirname)
  xml.at("//sourcecode | //xmlns:sourcecode") or return
  FileUtils.mkdir_p "#{dirname}/sourcecode"
  xml.xpath("//sourcecode | //xmlns:sourcecode").each_with_index do |s, i|
    filename = s["filename"] || sprintf("sourcecode-%04d.txt", i)
    export_output("#{dirname}/sourcecode/#{filename}",
                  clean_sourcecode(s.dup))
  end
end

#validate_format(options) ⇒ Object 



17
18
19
20
21
22
 
# File 'lib/metanorma/compile_validate.rb', line 17

def validate_format(options)
  unless options[:format] == :asciidoc
    Util.log("[metanorma] Error: Only source file format currently "\
             "supported is 'asciidoc'.", :fatal)
  end
end

#validate_options(options) ⇒ Object 



3
4
5
6
 
# File 'lib/metanorma/compile_validate.rb', line 3

def validate_options(options)
  validate_type(options)
  validate_format(options)
end

#validate_type(options) ⇒ Object 



8
9
10
11
12
13
14
15
 
# File 'lib/metanorma/compile_validate.rb', line 8

def validate_type(options)
  unless options[:type]
    Util.log("[metanorma] Error: Please specify a standard type: "\
             "#{@registry.supported_backends}.", :fatal)
  end
  stdtype = options[:type].to_sym
  load_flavor(stdtype)
end

#wrap_html(options, file_extension, outfilename) ⇒ Object 



101
102
103
104
105
106
107
108
 
# File 'lib/metanorma/compile.rb', line 101

def wrap_html(options, file_extension, outfilename)
  if options[:wrapper] && /html$/.match(file_extension)
    outfilename = outfilename.sub(/\.html$/, "")
    FileUtils.mkdir_p outfilename
    FileUtils.mv "#{outfilename}.html", outfilename
    FileUtils.mv "#{outfilename}_images", outfilename, force: true
  end
end

#xml_options_extract(file) ⇒ Object 



9
10
11
12
13
14
15
16
17
 
# File 'lib/metanorma/compile_options.rb', line 9

def xml_options_extract(file)
  xml = Nokogiri::XML(file, &:huge)
  if xml.root
    @registry.root_tags.each do |k, v|
      return { type: k } if v == xml.root.name
    end
  end
  {}
end