1 # SPDX-License-Identifier: GPL-2.0 1 # SPDX-License-Identifier: GPL-2.0
2 # Copyright 2019 Jonathan Corbet <corbet@lwn.ne 2 # Copyright 2019 Jonathan Corbet <corbet@lwn.net>
3 # 3 #
4 # Apply kernel-specific tweaks after the initi 4 # Apply kernel-specific tweaks after the initial document processing
5 # has been done. 5 # has been done.
6 # 6 #
7 from docutils import nodes 7 from docutils import nodes
8 import sphinx 8 import sphinx
9 from sphinx import addnodes 9 from sphinx import addnodes
10 from sphinx.errors import NoUri 10 from sphinx.errors import NoUri
11 import re 11 import re
12 from itertools import chain 12 from itertools import chain
13 13
14 # 14 #
15 # Python 2 lacks re.ASCII... 15 # Python 2 lacks re.ASCII...
16 # 16 #
17 try: 17 try:
18 ascii_p3 = re.ASCII 18 ascii_p3 = re.ASCII
19 except AttributeError: 19 except AttributeError:
20 ascii_p3 = 0 20 ascii_p3 = 0
21 21
22 # 22 #
23 # Regex nastiness. Of course. 23 # Regex nastiness. Of course.
24 # Try to identify "function()" that's not alre 24 # Try to identify "function()" that's not already marked up some
25 # other way. Sphinx doesn't like a lot of stu 25 # other way. Sphinx doesn't like a lot of stuff right after a
26 # :c:func: block (i.e. ":c:func:`mmap()`s" fla 26 # :c:func: block (i.e. ":c:func:`mmap()`s" flakes out), so the last
27 # bit tries to restrict matches to things that 27 # bit tries to restrict matches to things that won't create trouble.
28 # 28 #
29 RE_function = re.compile(r'\b(([a-zA-Z_]\w+)\( 29 RE_function = re.compile(r'\b(([a-zA-Z_]\w+)\(\))', flags=ascii_p3)
30 30
31 # 31 #
32 # Sphinx 2 uses the same :c:type role for stru 32 # Sphinx 2 uses the same :c:type role for struct, union, enum and typedef
33 # 33 #
34 RE_generic_type = re.compile(r'\b(struct|union 34 RE_generic_type = re.compile(r'\b(struct|union|enum|typedef)\s+([a-zA-Z_]\w+)',
35 flags=ascii_p3) 35 flags=ascii_p3)
36 36
37 # 37 #
38 # Sphinx 3 uses a different C role for each on 38 # Sphinx 3 uses a different C role for each one of struct, union, enum and
39 # typedef 39 # typedef
40 # 40 #
41 RE_struct = re.compile(r'\b(struct)\s+([a-zA-Z 41 RE_struct = re.compile(r'\b(struct)\s+([a-zA-Z_]\w+)', flags=ascii_p3)
42 RE_union = re.compile(r'\b(union)\s+([a-zA-Z_] 42 RE_union = re.compile(r'\b(union)\s+([a-zA-Z_]\w+)', flags=ascii_p3)
43 RE_enum = re.compile(r'\b(enum)\s+([a-zA-Z_]\w 43 RE_enum = re.compile(r'\b(enum)\s+([a-zA-Z_]\w+)', flags=ascii_p3)
44 RE_typedef = re.compile(r'\b(typedef)\s+([a-zA 44 RE_typedef = re.compile(r'\b(typedef)\s+([a-zA-Z_]\w+)', flags=ascii_p3)
45 45
46 # 46 #
47 # Detects a reference to a documentation page 47 # Detects a reference to a documentation page of the form Documentation/... with
48 # an optional extension 48 # an optional extension
49 # 49 #
50 RE_doc = re.compile(r'(\bDocumentation/)?((\.\ 50 RE_doc = re.compile(r'(\bDocumentation/)?((\.\./)*[\w\-/]+)\.(rst|txt)')
51 51
52 RE_namespace = re.compile(r'^\s*..\s*c:namespa 52 RE_namespace = re.compile(r'^\s*..\s*c:namespace::\s*(\S+)\s*$')
53 53
54 # 54 #
55 # Reserved C words that we should skip when cr 55 # Reserved C words that we should skip when cross-referencing
56 # 56 #
57 Skipnames = [ 'for', 'if', 'register', 'sizeof 57 Skipnames = [ 'for', 'if', 'register', 'sizeof', 'struct', 'unsigned' ]
58 58
59 59
60 # 60 #
61 # Many places in the docs refer to common syst 61 # Many places in the docs refer to common system calls. It is
62 # pointless to try to cross-reference them and 62 # pointless to try to cross-reference them and, as has been known
63 # to happen, somebody defining a function by t 63 # to happen, somebody defining a function by these names can lead
64 # to the creation of incorrect and confusing c 64 # to the creation of incorrect and confusing cross references. So
65 # just don't even try with these names. 65 # just don't even try with these names.
66 # 66 #
67 Skipfuncs = [ 'open', 'close', 'read', 'write' 67 Skipfuncs = [ 'open', 'close', 'read', 'write', 'fcntl', 'mmap',
68 'select', 'poll', 'fork', 'execv 68 'select', 'poll', 'fork', 'execve', 'clone', 'ioctl',
69 'socket' ] 69 'socket' ]
70 70
71 c_namespace = '' 71 c_namespace = ''
72 72
73 # 73 #
74 # Detect references to commits. 74 # Detect references to commits.
75 # 75 #
76 RE_git = re.compile(r'commit\s+(?P<rev>[0-9a-f 76 RE_git = re.compile(r'commit\s+(?P<rev>[0-9a-f]{12,40})(?:\s+\(".*?"\))?',
77 flags=re.IGNORECASE | re.DOTALL) 77 flags=re.IGNORECASE | re.DOTALL)
78 78
79 def markup_refs(docname, app, node): 79 def markup_refs(docname, app, node):
80 t = node.astext() 80 t = node.astext()
81 done = 0 81 done = 0
82 repl = [ ] 82 repl = [ ]
83 # 83 #
84 # Associate each regex with the function t 84 # Associate each regex with the function that will markup its matches
85 # 85 #
86 markup_func_sphinx2 = {RE_doc: markup_doc_ 86 markup_func_sphinx2 = {RE_doc: markup_doc_ref,
87 RE_function: markup 87 RE_function: markup_c_ref,
88 RE_generic_type: ma 88 RE_generic_type: markup_c_ref}
89 89
90 markup_func_sphinx3 = {RE_doc: markup_doc_ 90 markup_func_sphinx3 = {RE_doc: markup_doc_ref,
91 RE_function: markup 91 RE_function: markup_func_ref_sphinx3,
92 RE_struct: markup_c 92 RE_struct: markup_c_ref,
93 RE_union: markup_c_ 93 RE_union: markup_c_ref,
94 RE_enum: markup_c_r 94 RE_enum: markup_c_ref,
95 RE_typedef: markup_ 95 RE_typedef: markup_c_ref,
96 RE_git: markup_git} 96 RE_git: markup_git}
97 97
98 if sphinx.version_info[0] >= 3: 98 if sphinx.version_info[0] >= 3:
99 markup_func = markup_func_sphinx3 99 markup_func = markup_func_sphinx3
100 else: 100 else:
101 markup_func = markup_func_sphinx2 101 markup_func = markup_func_sphinx2
102 102
103 match_iterators = [regex.finditer(t) for r 103 match_iterators = [regex.finditer(t) for regex in markup_func]
104 # 104 #
105 # Sort all references by the starting posi 105 # Sort all references by the starting position in text
106 # 106 #
107 sorted_matches = sorted(chain(*match_itera 107 sorted_matches = sorted(chain(*match_iterators), key=lambda m: m.start())
108 for m in sorted_matches: 108 for m in sorted_matches:
109 # 109 #
110 # Include any text prior to match as a 110 # Include any text prior to match as a normal text node.
111 # 111 #
112 if m.start() > done: 112 if m.start() > done:
113 repl.append(nodes.Text(t[done:m.st 113 repl.append(nodes.Text(t[done:m.start()]))
114 114
115 # 115 #
116 # Call the function associated with th 116 # Call the function associated with the regex that matched this text and
117 # append its return to the text 117 # append its return to the text
118 # 118 #
119 repl.append(markup_func[m.re](docname, 119 repl.append(markup_func[m.re](docname, app, m))
120 120
121 done = m.end() 121 done = m.end()
122 if done < len(t): 122 if done < len(t):
123 repl.append(nodes.Text(t[done:])) 123 repl.append(nodes.Text(t[done:]))
124 return repl 124 return repl
125 125
126 # 126 #
127 # Keep track of cross-reference lookups that f 127 # Keep track of cross-reference lookups that failed so we don't have to
128 # do them again. 128 # do them again.
129 # 129 #
130 failed_lookups = { } 130 failed_lookups = { }
131 def failure_seen(target): 131 def failure_seen(target):
132 return (target) in failed_lookups 132 return (target) in failed_lookups
133 def note_failure(target): 133 def note_failure(target):
134 failed_lookups[target] = True 134 failed_lookups[target] = True
135 135
136 # 136 #
137 # In sphinx3 we can cross-reference to C macro 137 # In sphinx3 we can cross-reference to C macro and function, each one with its
138 # own C role, but both match the same regex, s 138 # own C role, but both match the same regex, so we try both.
139 # 139 #
140 def markup_func_ref_sphinx3(docname, app, matc 140 def markup_func_ref_sphinx3(docname, app, match):
141 cdom = app.env.domains['c'] 141 cdom = app.env.domains['c']
142 # 142 #
143 # Go through the dance of getting an xref 143 # Go through the dance of getting an xref out of the C domain
144 # 144 #
145 base_target = match.group(2) 145 base_target = match.group(2)
146 target_text = nodes.Text(match.group(0)) 146 target_text = nodes.Text(match.group(0))
147 xref = None 147 xref = None
148 possible_targets = [base_target] 148 possible_targets = [base_target]
149 # Check if this document has a namespace, 149 # Check if this document has a namespace, and if so, try
150 # cross-referencing inside it first. 150 # cross-referencing inside it first.
151 if c_namespace: 151 if c_namespace:
152 possible_targets.insert(0, c_namespace 152 possible_targets.insert(0, c_namespace + "." + base_target)
153 153
154 if base_target not in Skipnames: 154 if base_target not in Skipnames:
155 for target in possible_targets: 155 for target in possible_targets:
156 if (target not in Skipfuncs) and n 156 if (target not in Skipfuncs) and not failure_seen(target):
157 lit_text = nodes.literal(class 157 lit_text = nodes.literal(classes=['xref', 'c', 'c-func'])
158 lit_text += target_text 158 lit_text += target_text
159 pxref = addnodes.pending_xref( 159 pxref = addnodes.pending_xref('', refdomain = 'c',
160 160 reftype = 'function',
161 161 reftarget = target,
162 162 modname = None,
163 163 classname = None)
164 # 164 #
165 # XXX The Latex builder will t 165 # XXX The Latex builder will throw NoUri exceptions here,
166 # work around that by ignoring 166 # work around that by ignoring them.
167 # 167 #
168 try: 168 try:
169 xref = cdom.resolve_xref(a 169 xref = cdom.resolve_xref(app.env, docname, app.builder,
170 ' 170 'function', target, pxref,
171 l 171 lit_text)
172 except NoUri: 172 except NoUri:
173 xref = None 173 xref = None
174 174
175 if xref: 175 if xref:
176 return xref 176 return xref
177 note_failure(target) 177 note_failure(target)
178 178
179 return target_text 179 return target_text
180 180
181 def markup_c_ref(docname, app, match): 181 def markup_c_ref(docname, app, match):
182 class_str = {# Sphinx 2 only 182 class_str = {# Sphinx 2 only
183 RE_function: 'c-func', 183 RE_function: 'c-func',
184 RE_generic_type: 'c-type', 184 RE_generic_type: 'c-type',
185 # Sphinx 3+ only 185 # Sphinx 3+ only
186 RE_struct: 'c-struct', 186 RE_struct: 'c-struct',
187 RE_union: 'c-union', 187 RE_union: 'c-union',
188 RE_enum: 'c-enum', 188 RE_enum: 'c-enum',
189 RE_typedef: 'c-type', 189 RE_typedef: 'c-type',
190 } 190 }
191 reftype_str = {# Sphinx 2 only 191 reftype_str = {# Sphinx 2 only
192 RE_function: 'function', 192 RE_function: 'function',
193 RE_generic_type: 'type', 193 RE_generic_type: 'type',
194 # Sphinx 3+ only 194 # Sphinx 3+ only
195 RE_struct: 'struct', 195 RE_struct: 'struct',
196 RE_union: 'union', 196 RE_union: 'union',
197 RE_enum: 'enum', 197 RE_enum: 'enum',
198 RE_typedef: 'type', 198 RE_typedef: 'type',
199 } 199 }
200 200
201 cdom = app.env.domains['c'] 201 cdom = app.env.domains['c']
202 # 202 #
203 # Go through the dance of getting an xref 203 # Go through the dance of getting an xref out of the C domain
204 # 204 #
205 base_target = match.group(2) 205 base_target = match.group(2)
206 target_text = nodes.Text(match.group(0)) 206 target_text = nodes.Text(match.group(0))
207 xref = None 207 xref = None
208 possible_targets = [base_target] 208 possible_targets = [base_target]
209 # Check if this document has a namespace, 209 # Check if this document has a namespace, and if so, try
210 # cross-referencing inside it first. 210 # cross-referencing inside it first.
211 if c_namespace: 211 if c_namespace:
212 possible_targets.insert(0, c_namespace 212 possible_targets.insert(0, c_namespace + "." + base_target)
213 213
214 if base_target not in Skipnames: 214 if base_target not in Skipnames:
215 for target in possible_targets: 215 for target in possible_targets:
216 if not (match.re == RE_function an 216 if not (match.re == RE_function and target in Skipfuncs):
217 lit_text = nodes.literal(class 217 lit_text = nodes.literal(classes=['xref', 'c', class_str[match.re]])
218 lit_text += target_text 218 lit_text += target_text
219 pxref = addnodes.pending_xref( 219 pxref = addnodes.pending_xref('', refdomain = 'c',
220 220 reftype = reftype_str[match.re],
221 221 reftarget = target, modname = None,
222 222 classname = None)
223 # 223 #
224 # XXX The Latex builder will t 224 # XXX The Latex builder will throw NoUri exceptions here,
225 # work around that by ignoring 225 # work around that by ignoring them.
226 # 226 #
227 try: 227 try:
228 xref = cdom.resolve_xref(a 228 xref = cdom.resolve_xref(app.env, docname, app.builder,
229 r 229 reftype_str[match.re], target, pxref,
230 l 230 lit_text)
231 except NoUri: 231 except NoUri:
232 xref = None 232 xref = None
233 233
234 if xref: 234 if xref:
235 return xref 235 return xref
236 236
237 return target_text 237 return target_text
238 238
239 # 239 #
240 # Try to replace a documentation reference of 240 # Try to replace a documentation reference of the form Documentation/... with a
241 # cross reference to that page 241 # cross reference to that page
242 # 242 #
243 def markup_doc_ref(docname, app, match): 243 def markup_doc_ref(docname, app, match):
244 stddom = app.env.domains['std'] 244 stddom = app.env.domains['std']
245 # 245 #
246 # Go through the dance of getting an xref 246 # Go through the dance of getting an xref out of the std domain
247 # 247 #
248 absolute = match.group(1) 248 absolute = match.group(1)
249 target = match.group(2) 249 target = match.group(2)
250 if absolute: 250 if absolute:
251 target = "/" + target 251 target = "/" + target
252 xref = None 252 xref = None
253 pxref = addnodes.pending_xref('', refdomai 253 pxref = addnodes.pending_xref('', refdomain = 'std', reftype = 'doc',
254 reftarget = 254 reftarget = target, modname = None,
255 classname = 255 classname = None, refexplicit = False)
256 # 256 #
257 # XXX The Latex builder will throw NoUri e 257 # XXX The Latex builder will throw NoUri exceptions here,
258 # work around that by ignoring them. 258 # work around that by ignoring them.
259 # 259 #
260 try: 260 try:
261 xref = stddom.resolve_xref(app.env, do 261 xref = stddom.resolve_xref(app.env, docname, app.builder, 'doc',
262 target, pxr 262 target, pxref, None)
263 except NoUri: 263 except NoUri:
264 xref = None 264 xref = None
265 # 265 #
266 # Return the xref if we got it; otherwise 266 # Return the xref if we got it; otherwise just return the plain text.
267 # 267 #
268 if xref: 268 if xref:
269 return xref 269 return xref
270 else: 270 else:
271 return nodes.Text(match.group(0)) 271 return nodes.Text(match.group(0))
272 272
273 def get_c_namespace(app, docname): 273 def get_c_namespace(app, docname):
274 source = app.env.doc2path(docname) 274 source = app.env.doc2path(docname)
275 with open(source) as f: 275 with open(source) as f:
276 for l in f: 276 for l in f:
277 match = RE_namespace.search(l) 277 match = RE_namespace.search(l)
278 if match: 278 if match:
279 return match.group(1) 279 return match.group(1)
280 return '' 280 return ''
281 281
282 def markup_git(docname, app, match): 282 def markup_git(docname, app, match):
283 # While we could probably assume that we a 283 # While we could probably assume that we are running in a git
284 # repository, we can't know for sure, so l 284 # repository, we can't know for sure, so let's just mechanically
285 # turn them into git.kernel.org links with 285 # turn them into git.kernel.org links without checking their
286 # validity. (Maybe we can do something in 286 # validity. (Maybe we can do something in the future to warn about
287 # these references if this is explicitly r 287 # these references if this is explicitly requested.)
288 text = match.group(0) 288 text = match.group(0)
289 rev = match.group('rev') 289 rev = match.group('rev')
290 return nodes.reference('', nodes.Text(text 290 return nodes.reference('', nodes.Text(text),
291 refuri=f'https://git.kernel.org/torval 291 refuri=f'https://git.kernel.org/torvalds/c/{rev}')
292 292
293 def auto_markup(app, doctree, name): 293 def auto_markup(app, doctree, name):
294 global c_namespace 294 global c_namespace
295 c_namespace = get_c_namespace(app, name) 295 c_namespace = get_c_namespace(app, name)
296 def text_but_not_a_reference(node): 296 def text_but_not_a_reference(node):
297 # The nodes.literal test catches ``lit 297 # The nodes.literal test catches ``literal text``, its purpose is to
298 # avoid adding cross-references to fun 298 # avoid adding cross-references to functions that have been explicitly
299 # marked with cc:func:. 299 # marked with cc:func:.
300 if not isinstance(node, nodes.Text) or 300 if not isinstance(node, nodes.Text) or isinstance(node.parent, nodes.literal):
301 return False 301 return False
302 302
303 child_of_reference = False 303 child_of_reference = False
304 parent = node.parent 304 parent = node.parent
305 while parent: 305 while parent:
306 if isinstance(parent, nodes.Refere 306 if isinstance(parent, nodes.Referential):
307 child_of_reference = True 307 child_of_reference = True
308 break 308 break
309 parent = parent.parent 309 parent = parent.parent
310 return not child_of_reference 310 return not child_of_reference
311 311
312 # 312 #
313 # This loop could eventually be improved o 313 # This loop could eventually be improved on. Someday maybe we
314 # want a proper tree traversal with a lot 314 # want a proper tree traversal with a lot of awareness of which
315 # kinds of nodes to prune. But this works 315 # kinds of nodes to prune. But this works well for now.
316 # 316 #
317 for para in doctree.traverse(nodes.paragra 317 for para in doctree.traverse(nodes.paragraph):
318 for node in para.traverse(condition=te 318 for node in para.traverse(condition=text_but_not_a_reference):
319 node.parent.replace(node, markup_r 319 node.parent.replace(node, markup_refs(name, app, node))
320 320
321 def setup(app): 321 def setup(app):
322 app.connect('doctree-resolved', auto_marku 322 app.connect('doctree-resolved', auto_markup)
323 return { 323 return {
324 'parallel_read_safe': True, 324 'parallel_read_safe': True,
325 'parallel_write_safe': True, 325 'parallel_write_safe': True,
326 } 326 }
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.