1 #!/usr/bin/env python3
2 # SPDX-License-Identifier: GPL-2.0
3 # -*- coding: utf-8; mode: python -*-
4
5 """
6 Script to auto generate the documentation for Netlink specifications.
7
8 :copyright: Copyright (C) 2023 Breno Leitao <leitao@debian.org>
9 :license: GPL Version 2, June 1991 see linux/COPYING for details.
10
11 This script performs extensive parsing to the Linux kernel's netlink YAML
12 spec files, in an effort to avoid needing to heavily mark up the original
13 YAML file.
14
15 This code is split in three big parts:
16 1) RST formatters: Use to convert a string to a RST output
17 2) Parser helpers: Functions to parse the YAML data structure
18 3) Main function and small helpers
19 """
20
21 from typing import Any, Dict, List
22 import os.path
23 import sys
24 import argparse
25 import logging
26 import yaml
27
28
29 SPACE_PER_LEVEL = 4
30
31
32 # RST Formatters
33 # ==============
34 def headroom(level: int) -> str:
35 """Return space to format"""
36 return " " * (level * SPACE_PER_LEVEL)
37
38
39 def bold(text: str) -> str:
40 """Format bold text"""
41 return f"**{text}**"
42
43
44 def inline(text: str) -> str:
45 """Format inline text"""
46 return f"``{text}``"
47
48
49 def sanitize(text: str) -> str:
50 """Remove newlines and multiple spaces"""
51 # This is useful for some fields that are spread across multiple lines
52 return str(text).replace("\n", " ").strip()
53
54
55 def rst_fields(key: str, value: str, level: int = 0) -> str:
56 """Return a RST formatted field"""
57 return headroom(level) + f":{key}: {value}"
58
59
60 def rst_definition(key: str, value: Any, level: int = 0) -> str:
61 """Format a single rst definition"""
62 return headroom(level) + key + "\n" + headroom(level + 1) + str(value)
63
64
65 def rst_paragraph(paragraph: str, level: int = 0) -> str:
66 """Return a formatted paragraph"""
67 return headroom(level) + paragraph
68
69
70 def rst_bullet(item: str, level: int = 0) -> str:
71 """Return a formatted a bullet"""
72 return headroom(level) + f"- {item}"
73
74
75 def rst_subsection(title: str) -> str:
76 """Add a sub-section to the document"""
77 return f"{title}\n" + "-" * len(title)
78
79
80 def rst_subsubsection(title: str) -> str:
81 """Add a sub-sub-section to the document"""
82 return f"{title}\n" + "~" * len(title)
83
84
85 def rst_section(namespace: str, prefix: str, title: str) -> str:
86 """Add a section to the document"""
87 return f".. _{namespace}-{prefix}-{title}:\n\n{title}\n" + "=" * len(title)
88
89
90 def rst_subtitle(title: str) -> str:
91 """Add a subtitle to the document"""
92 return "\n" + "-" * len(title) + f"\n{title}\n" + "-" * len(title) + "\n\n"
93
94
95 def rst_title(title: str) -> str:
96 """Add a title to the document"""
97 return "=" * len(title) + f"\n{title}\n" + "=" * len(title) + "\n\n"
98
99
100 def rst_list_inline(list_: List[str], level: int = 0) -> str:
101 """Format a list using inlines"""
102 return headroom(level) + "[" + ", ".join(inline(i) for i in list_) + "]"
103
104
105 def rst_ref(namespace: str, prefix: str, name: str) -> str:
106 """Add a hyperlink to the document"""
107 mappings = {'enum': 'definition',
108 'fixed-header': 'definition',
109 'nested-attributes': 'attribute-set',
110 'struct': 'definition'}
111 if prefix in mappings:
112 prefix = mappings[prefix]
113 return f":ref:`{namespace}-{prefix}-{name}`"
114
115
116 def rst_header() -> str:
117 """The headers for all the auto generated RST files"""
118 lines = []
119
120 lines.append(rst_paragraph(".. SPDX-License-Identifier: GPL-2.0"))
121 lines.append(rst_paragraph(".. NOTE: This document was auto-generated.\n\n"))
122
123 return "\n".join(lines)
124
125
126 def rst_toctree(maxdepth: int = 2) -> str:
127 """Generate a toctree RST primitive"""
128 lines = []
129
130 lines.append(".. toctree::")
131 lines.append(f" :maxdepth: {maxdepth}\n\n")
132
133 return "\n".join(lines)
134
135
136 def rst_label(title: str) -> str:
137 """Return a formatted label"""
138 return f".. _{title}:\n\n"
139
140
141 # Parsers
142 # =======
143
144
145 def parse_mcast_group(mcast_group: List[Dict[str, Any]]) -> str:
146 """Parse 'multicast' group list and return a formatted string"""
147 lines = []
148 for group in mcast_group:
149 lines.append(rst_bullet(group["name"]))
150
151 return "\n".join(lines)
152
153
154 def parse_do(do_dict: Dict[str, Any], level: int = 0) -> str:
155 """Parse 'do' section and return a formatted string"""
156 lines = []
157 for key in do_dict.keys():
158 lines.append(rst_paragraph(bold(key), level + 1))
159 if key in ['request', 'reply']:
160 lines.append(parse_do_attributes(do_dict[key], level + 1) + "\n")
161 else:
162 lines.append(headroom(level + 2) + do_dict[key] + "\n")
163
164 return "\n".join(lines)
165
166
167 def parse_do_attributes(attrs: Dict[str, Any], level: int = 0) -> str:
168 """Parse 'attributes' section"""
169 if "attributes" not in attrs:
170 return ""
171 lines = [rst_fields("attributes", rst_list_inline(attrs["attributes"]), level + 1)]
172
173 return "\n".join(lines)
174
175
176 def parse_operations(operations: List[Dict[str, Any]], namespace: str) -> str:
177 """Parse operations block"""
178 preprocessed = ["name", "doc", "title", "do", "dump", "flags"]
179 linkable = ["fixed-header", "attribute-set"]
180 lines = []
181
182 for operation in operations:
183 lines.append(rst_section(namespace, 'operation', operation["name"]))
184 lines.append(rst_paragraph(operation["doc"]) + "\n")
185
186 for key in operation.keys():
187 if key in preprocessed:
188 # Skip the special fields
189 continue
190 value = operation[key]
191 if key in linkable:
192 value = rst_ref(namespace, key, value)
193 lines.append(rst_fields(key, value, 0))
194 if 'flags' in operation:
195 lines.append(rst_fields('flags', rst_list_inline(operation['flags'])))
196
197 if "do" in operation:
198 lines.append(rst_paragraph(":do:", 0))
199 lines.append(parse_do(operation["do"], 0))
200 if "dump" in operation:
201 lines.append(rst_paragraph(":dump:", 0))
202 lines.append(parse_do(operation["dump"], 0))
203
204 # New line after fields
205 lines.append("\n")
206
207 return "\n".join(lines)
208
209
210 def parse_entries(entries: List[Dict[str, Any]], level: int) -> str:
211 """Parse a list of entries"""
212 ignored = ["pad"]
213 lines = []
214 for entry in entries:
215 if isinstance(entry, dict):
216 # entries could be a list or a dictionary
217 field_name = entry.get("name", "")
218 if field_name in ignored:
219 continue
220 type_ = entry.get("type")
221 if type_:
222 field_name += f" ({inline(type_)})"
223 lines.append(
224 rst_fields(field_name, sanitize(entry.get("doc", "")), level)
225 )
226 elif isinstance(entry, list):
227 lines.append(rst_list_inline(entry, level))
228 else:
229 lines.append(rst_bullet(inline(sanitize(entry)), level))
230
231 lines.append("\n")
232 return "\n".join(lines)
233
234
235 def parse_definitions(defs: Dict[str, Any], namespace: str) -> str:
236 """Parse definitions section"""
237 preprocessed = ["name", "entries", "members"]
238 ignored = ["render-max"] # This is not printed
239 lines = []
240
241 for definition in defs:
242 lines.append(rst_section(namespace, 'definition', definition["name"]))
243 for k in definition.keys():
244 if k in preprocessed + ignored:
245 continue
246 lines.append(rst_fields(k, sanitize(definition[k]), 0))
247
248 # Field list needs to finish with a new line
249 lines.append("\n")
250 if "entries" in definition:
251 lines.append(rst_paragraph(":entries:", 0))
252 lines.append(parse_entries(definition["entries"], 1))
253 if "members" in definition:
254 lines.append(rst_paragraph(":members:", 0))
255 lines.append(parse_entries(definition["members"], 1))
256
257 return "\n".join(lines)
258
259
260 def parse_attr_sets(entries: List[Dict[str, Any]], namespace: str) -> str:
261 """Parse attribute from attribute-set"""
262 preprocessed = ["name", "type"]
263 linkable = ["enum", "nested-attributes", "struct", "sub-message"]
264 ignored = ["checks"]
265 lines = []
266
267 for entry in entries:
268 lines.append(rst_section(namespace, 'attribute-set', entry["name"]))
269 for attr in entry["attributes"]:
270 type_ = attr.get("type")
271 attr_line = attr["name"]
272 if type_:
273 # Add the attribute type in the same line
274 attr_line += f" ({inline(type_)})"
275
276 lines.append(rst_subsubsection(attr_line))
277
278 for k in attr.keys():
279 if k in preprocessed + ignored:
280 continue
281 if k in linkable:
282 value = rst_ref(namespace, k, attr[k])
283 else:
284 value = sanitize(attr[k])
285 lines.append(rst_fields(k, value, 0))
286 lines.append("\n")
287
288 return "\n".join(lines)
289
290
291 def parse_sub_messages(entries: List[Dict[str, Any]], namespace: str) -> str:
292 """Parse sub-message definitions"""
293 lines = []
294
295 for entry in entries:
296 lines.append(rst_section(namespace, 'sub-message', entry["name"]))
297 for fmt in entry["formats"]:
298 value = fmt["value"]
299
300 lines.append(rst_bullet(bold(value)))
301 for attr in ['fixed-header', 'attribute-set']:
302 if attr in fmt:
303 lines.append(rst_fields(attr,
304 rst_ref(namespace, attr, fmt[attr]),
305 1))
306 lines.append("\n")
307
308 return "\n".join(lines)
309
310
311 def parse_yaml(obj: Dict[str, Any]) -> str:
312 """Format the whole YAML into a RST string"""
313 lines = []
314
315 # Main header
316
317 lines.append(rst_header())
318
319 family = obj['name']
320
321 title = f"Family ``{family}`` netlink specification"
322 lines.append(rst_title(title))
323 lines.append(rst_paragraph(".. contents:: :depth: 3\n"))
324
325 if "doc" in obj:
326 lines.append(rst_subtitle("Summary"))
327 lines.append(rst_paragraph(obj["doc"], 0))
328
329 # Operations
330 if "operations" in obj:
331 lines.append(rst_subtitle("Operations"))
332 lines.append(parse_operations(obj["operations"]["list"], family))
333
334 # Multicast groups
335 if "mcast-groups" in obj:
336 lines.append(rst_subtitle("Multicast groups"))
337 lines.append(parse_mcast_group(obj["mcast-groups"]["list"]))
338
339 # Definitions
340 if "definitions" in obj:
341 lines.append(rst_subtitle("Definitions"))
342 lines.append(parse_definitions(obj["definitions"], family))
343
344 # Attributes set
345 if "attribute-sets" in obj:
346 lines.append(rst_subtitle("Attribute sets"))
347 lines.append(parse_attr_sets(obj["attribute-sets"], family))
348
349 # Sub-messages
350 if "sub-messages" in obj:
351 lines.append(rst_subtitle("Sub-messages"))
352 lines.append(parse_sub_messages(obj["sub-messages"], family))
353
354 return "\n".join(lines)
355
356
357 # Main functions
358 # ==============
359
360
361 def parse_arguments() -> argparse.Namespace:
362 """Parse arguments from user"""
363 parser = argparse.ArgumentParser(description="Netlink RST generator")
364
365 parser.add_argument("-v", "--verbose", action="store_true")
366 parser.add_argument("-o", "--output", help="Output file name")
367
368 # Index and input are mutually exclusive
369 group = parser.add_mutually_exclusive_group()
370 group.add_argument(
371 "-x", "--index", action="store_true", help="Generate the index page"
372 )
373 group.add_argument("-i", "--input", help="YAML file name")
374
375 args = parser.parse_args()
376
377 if args.verbose:
378 logging.basicConfig(level=logging.DEBUG)
379
380 if args.input and not os.path.isfile(args.input):
381 logging.warning("%s is not a valid file.", args.input)
382 sys.exit(-1)
383
384 if not args.output:
385 logging.error("No output file specified.")
386 sys.exit(-1)
387
388 if os.path.isfile(args.output):
389 logging.debug("%s already exists. Overwriting it.", args.output)
390
391 return args
392
393
394 def parse_yaml_file(filename: str) -> str:
395 """Transform the YAML specified by filename into a rst-formmated string"""
396 with open(filename, "r", encoding="utf-8") as spec_file:
397 yaml_data = yaml.safe_load(spec_file)
398 content = parse_yaml(yaml_data)
399
400 return content
401
402
403 def write_to_rstfile(content: str, filename: str) -> None:
404 """Write the generated content into an RST file"""
405 logging.debug("Saving RST file to %s", filename)
406
407 with open(filename, "w", encoding="utf-8") as rst_file:
408 rst_file.write(content)
409
410
411 def generate_main_index_rst(output: str) -> None:
412 """Generate the `networking_spec/index` content and write to the file"""
413 lines = []
414
415 lines.append(rst_header())
416 lines.append(rst_label("specs"))
417 lines.append(rst_title("Netlink Family Specifications"))
418 lines.append(rst_toctree(1))
419
420 index_dir = os.path.dirname(output)
421 logging.debug("Looking for .rst files in %s", index_dir)
422 for filename in sorted(os.listdir(index_dir)):
423 if not filename.endswith(".rst") or filename == "index.rst":
424 continue
425 lines.append(f" {filename.replace('.rst', '')}\n")
426
427 logging.debug("Writing an index file at %s", output)
428 write_to_rstfile("".join(lines), output)
429
430
431 def main() -> None:
432 """Main function that reads the YAML files and generates the RST files"""
433
434 args = parse_arguments()
435
436 if args.input:
437 logging.debug("Parsing %s", args.input)
438 try:
439 content = parse_yaml_file(os.path.join(args.input))
440 except Exception as exception:
441 logging.warning("Failed to parse %s.", args.input)
442 logging.warning(exception)
443 sys.exit(-1)
444
445 write_to_rstfile(content, args.output)
446
447 if args.index:
448 # Generate the index RST file
449 generate_main_index_rst(args.output)
450
451
452 if __name__ == "__main__":
453 main()
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.