1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
|
##
# This file contains stuff stolen outright from:
#
# rtags.rb -
# ruby-lex.rb - ruby lexcal analyzer
# ruby-token.rb - ruby tokens
# by Keiju ISHITSUKA (Nippon Rational Inc.)
#
require 'rdoc/ruby_token'
require 'rdoc/ruby_lex'
require 'rdoc/code_objects'
require 'rdoc/tokenstream'
require 'rdoc/markup/preprocess'
require 'rdoc/parser'
require 'rdoc/parser/ruby_tools'
$TOKEN_DEBUG ||= nil
##
# Extracts code elements from a source file returning a TopLevel object
# containing the constituent file elements.
#
# This file is based on rtags
#
# RubyParser understands how to document:
# * classes
# * modules
# * methods
# * constants
# * aliases
# * private, public, protected
# * private_class_function, public_class_function
# * module_function
# * attr, attr_reader, attr_writer, attr_accessor
# * extra accessors given on the command line
# * metaprogrammed methods
# * require
# * include
#
# == Method Arguments
#
#--
# NOTE: I don't think this works, needs tests, remove the paragraph following
# this block when known to work
#
# The parser extracts the arguments from the method definition. You can
# override this with a custom argument definition using the :args: directive:
#
# ##
# # This method tries over and over until it is tired
#
# def go_go_go(thing_to_try, tries = 10) # :args: thing_to_try
# puts thing_to_try
# go_go_go thing_to_try, tries - 1
# end
#
# If you have a more-complex set of overrides you can use the :call-seq:
# directive:
#++
#
# The parser extracts the arguments from the method definition. You can
# override this with a custom argument definition using the :call-seq:
# directive:
#
# ##
# # This method can be called with a range or an offset and length
# #
# # :call-seq:
# # my_method(Range)
# # my_method(offset, length)
#
# def my_method(*args)
# end
#
# The parser extracts +yield+ expressions from method bodies to gather the
# yielded argument names. If your method manually calls a block instead of
# yielding or you want to override the discovered argument names use
# the :yields: directive:
#
# ##
# # My method is awesome
#
# def my_method(&block) # :yields: happy, times
# block.call 1, 2
# end
#
# == Metaprogrammed Methods
#
# To pick up a metaprogrammed method, the parser looks for a comment starting
# with '##' before an identifier:
#
# ##
# # This is a meta-programmed method!
#
# add_my_method :meta_method, :arg1, :arg2
#
# The parser looks at the token after the identifier to determine the name, in
# this example, :meta_method. If a name cannot be found, a warning is printed
# and 'unknown is used.
#
# You can force the name of a method using the :method: directive:
#
# ##
# # :method: woo_hoo!
#
# By default, meta-methods are instance methods. To indicate that a method is
# a singleton method instead use the :singleton-method: directive:
#
# ##
# # :singleton-method:
#
# You can also use the :singleton-method: directive with a name:
#
# ##
# # :singleton-method: woo_hoo!
#
# Additionally you can mark a method as an attribute by
# using :attr:, :attr_reader:, :attr_writer: or :attr_accessor:. Just like
# for :method:, the name is optional.
#
# ##
# # :attr_reader: my_attr_name
#
# == Hidden methods and attributes
#
# You can provide documentation for methods that don't appear using
# the :method:, :singleton-method: and :attr: directives:
#
# ##
# # :attr_writer: ghost_writer
# # There is an attribute here, but you can't see it!
#
# ##
# # :method: ghost_method
# # There is a method here, but you can't see it!
#
# ##
# # this is a comment for a regular method
#
# def regular_method() end
#
# Note that by default, the :method: directive will be ignored if there is a
# standard rdocable item following it.
class RDoc::Parser::Ruby < RDoc::Parser
parse_files_matching(/\.(?:rbw?|rdoc)\z/)
include RDoc::RubyToken
include RDoc::TokenStream
include RDoc::Parser::RubyTools
##
# RDoc::NormalClass type
NORMAL = "::"
##
# RDoc::SingleClass type
SINGLE = "<<"
def initialize(top_level, file_name, content, options, stats)
super
@size = 0
@token_listeners = nil
@scanner = RDoc::RubyLex.new content, @options
@scanner.exception_on_syntax_error = false
@prev_seek = nil
reset
end
##
# Look for the first comment in a file that isn't a shebang line.
def collect_first_comment
skip_tkspace
comment = ''
first_line = true
tk = get_tk
while TkCOMMENT === tk
if first_line and tk.text =~ /\A#!/ then
skip_tkspace
tk = get_tk
elsif first_line and tk.text =~ /\A#\s*-\*-/ then
first_line = false
skip_tkspace
tk = get_tk
else
first_line = false
comment << tk.text << "\n"
tk = get_tk
if TkNL === tk then
skip_tkspace false
tk = get_tk
end
end
end
unget_tk tk
comment
end
def error(msg)
msg = make_message msg
$stderr.puts msg
exit(1)
end
##
# Look for a 'call-seq' in the comment, and override the normal parameter
# stuff
def extract_call_seq(comment, meth)
if comment.sub!(/:?call-seq:(.*?)^\s*\#?\s*$/m, '') then
seq = $1
seq.gsub!(/^\s*\#\s*/, '')
meth.call_seq = seq
end
meth
end
def get_bool
skip_tkspace
tk = get_tk
case tk
when TkTRUE
true
when TkFALSE, TkNIL
false
else
unget_tk tk
true
end
end
##
# Look for the name of a class of module (optionally with a leading :: or
# with :: separated named) and return the ultimate name and container
def get_class_or_module(container)
skip_tkspace
name_t = get_tk
# class ::A -> A is in the top level
case name_t
when TkCOLON2, TkCOLON3 then # bug
name_t = get_tk
container = @top_level
end
skip_tkspace false
while TkCOLON2 === peek_tk do
prev_container = container
container = container.find_module_named name_t.name
unless container then
container = prev_container.add_module RDoc::NormalModule, name_t.name
end
get_tk
name_t = get_tk
end
skip_tkspace false
return [container, name_t]
end
##
# Return a superclass, which can be either a constant of an expression
def get_class_specification
tk = get_tk
return "self" if TkSELF === tk
res = ""
while TkCOLON2 === tk or TkCOLON3 === tk or TkCONSTANT === tk do
res += tk.name
tk = get_tk
end
unget_tk(tk)
skip_tkspace false
get_tkread # empty out read buffer
tk = get_tk
case tk
when TkNL, TkCOMMENT, TkSEMICOLON then
unget_tk(tk)
return res
end
res += parse_call_parameters(tk)
res
end
##
# Parse a constant, which might be qualified by one or more class or module
# names
def get_constant
res = ""
skip_tkspace false
tk = get_tk
while TkCOLON2 === tk or TkCOLON3 === tk or TkCONSTANT === tk do
res += tk.name
tk = get_tk
end
# if res.empty?
# warn("Unexpected token #{tk} in constant")
# end
unget_tk(tk)
res
end
##
# Get a constant that may be surrounded by parens
def get_constant_with_optional_parens
skip_tkspace false
nest = 0
while TkLPAREN === (tk = peek_tk) or TkfLPAREN === tk do
get_tk
skip_tkspace
nest += 1
end
name = get_constant
while nest > 0
skip_tkspace
tk = get_tk
nest -= 1 if TkRPAREN === tk
end
name
end
def get_symbol_or_name
tk = get_tk
case tk
when TkSYMBOL then
text = tk.text.sub(/^:/, '')
if TkASSIGN === peek_tk then
get_tk
text << '='
end
text
when TkId, TkOp then
tk.name
when TkSTRING, TkDSTRING then
tk.text
else
raise RDoc::Error, "Name or symbol expected (got #{tk})"
end
end
##
# Look for directives in a normal comment block:
#
# # :stopdoc:
# # Don't display comment from this point forward
#
# This routine modifies it's parameter
def look_for_directives_in(context, comment)
preprocess = RDoc::Markup::PreProcess.new(@file_name,
@options.rdoc_include)
preprocess.handle(comment) do |directive, param|
case directive
when 'enddoc' then
throw :enddoc
when 'main' then
@options.main_page = param
''
when 'method', 'singleton-method',
'attr', 'attr_accessor', 'attr_reader', 'attr_writer' then
false # handled elsewhere
when 'section' then
context.set_current_section(param, comment)
comment.replace ''
break
when 'startdoc' then
context.start_doc
context.force_documentation = true
''
when 'stopdoc' then
context.stop_doc
''
when 'title' then
@options.title = param
''
else
warn "Unrecognized directive '#{directive}'"
false
end
end
remove_private_comments comment
end
##
# Adds useful info about the parser to +message+
def make_message message
prefix = "\n#{@file_name}:"
prefix << "#{@scanner.line_no}:#{@scanner.char_no}:" if @scanner
"#{prefix} #{message}"
end
##
# Creates an RDoc::Attr for the name following +tk+, setting the comment to
# +comment+.
def parse_attr(context, single, tk, comment)
args = parse_symbol_arg 1
if args.size > 0
name = args[0]
rw = "R"
skip_tkspace false
tk = get_tk
if TkCOMMA === tk then
rw = "RW" if get_bool
else
unget_tk tk
end
att = RDoc::Attr.new get_tkread, name, rw, comment
read_documentation_modifiers att, RDoc::ATTR_MODIFIERS
if att.document_self
context.add_attribute(att)
end
else
warn("'attr' ignored - looks like a variable")
end
end
##
# Creates an RDoc::Attr for each attribute listed after +tk+, setting the
# comment for each to +comment+.
def parse_attr_accessor(context, single, tk, comment)
args = parse_symbol_arg
read = get_tkread
rw = "?"
# TODO If nodoc is given, don't document any of them
tmp = RDoc::CodeObject.new
read_documentation_modifiers tmp, RDoc::ATTR_MODIFIERS
return unless tmp.document_self
case tk.name
when "attr_reader" then rw = "R"
when "attr_writer" then rw = "W"
when "attr_accessor" then rw = "RW"
else
rw = '?'
end
for name in args
att = RDoc::Attr.new get_tkread, name, rw, comment
context.add_attribute att
end
end
def parse_alias(context, single, tk, comment)
skip_tkspace
if TkLPAREN === peek_tk then
get_tk
skip_tkspace
end
new_name = get_symbol_or_name
@scanner.instance_eval { @lex_state = EXPR_FNAME }
skip_tkspace
if TkCOMMA === peek_tk then
get_tk
skip_tkspace
end
begin
old_name = get_symbol_or_name
rescue RDoc::Error
return
end
al = RDoc::Alias.new get_tkread, old_name, new_name, comment
read_documentation_modifiers al, RDoc::ATTR_MODIFIERS
context.add_alias al if al.document_self
end
def parse_call_parameters(tk)
end_token = case tk
when TkLPAREN, TkfLPAREN
TkRPAREN
when TkRPAREN
return ""
else
TkNL
end
nest = 0
loop do
case tk
when TkSEMICOLON
break
when TkLPAREN, TkfLPAREN
nest += 1
when end_token
if end_token == TkRPAREN
nest -= 1
break if @scanner.lex_state == EXPR_END and nest <= 0
else
break unless @scanner.continue
end
when TkCOMMENT
unget_tk(tk)
break
when nil then
break
end
tk = get_tk
end
res = get_tkread.tr("\n", " ").strip
res = "" if res == ";"
res
end
def parse_class(container, single, tk, comment)
container, name_t = get_class_or_module container
case name_t
when TkCONSTANT
name = name_t.name
superclass = "Object"
if TkLT === peek_tk then
get_tk
skip_tkspace
superclass = get_class_specification
superclass = "<unknown>" if superclass.empty?
end
cls_type = single == SINGLE ? RDoc::SingleClass : RDoc::NormalClass
cls = container.add_class cls_type, name, superclass
read_documentation_modifiers cls, RDoc::CLASS_MODIFIERS
cls.record_location @top_level
cls.comment = comment
@stats.add_class cls
parse_statements cls
when TkLSHFT
case name = get_class_specification
when "self", container.name
parse_statements container, SINGLE
else
other = RDoc::TopLevel.find_class_named name
unless other then
other = container.add_module RDoc::NormalModule, name
other.record_location @top_level
other.comment = comment
end
@stats.add_class other
read_documentation_modifiers other, RDoc::CLASS_MODIFIERS
parse_statements(other, SINGLE)
end
else
warn("Expected class name or '<<'. Got #{name_t.class}: #{name_t.text.inspect}")
end
end
def parse_constant(container, tk, comment)
name = tk.name
skip_tkspace false
eq_tk = get_tk
unless TkASSIGN === eq_tk then
unget_tk eq_tk
return
end
nest = 0
get_tkread
tk = get_tk
if TkGT === tk then
unget_tk tk
unget_tk eq_tk
return
end
rhs_name = ''
loop do
case tk
when TkSEMICOLON then
break
when TkLPAREN, TkfLPAREN, TkLBRACE, TkLBRACK, TkDO, TkIF, TkUNLESS,
TkCASE then
nest += 1
when TkRPAREN, TkRBRACE, TkRBRACK, TkEND then
nest -= 1
when TkCOMMENT then
if nest <= 0 && @scanner.lex_state == EXPR_END
unget_tk tk
break
end
when TkCONSTANT then
rhs_name << tk.name
if nest <= 0 and TkNL === peek_tk then
mod = if rhs_name =~ /^::/ then
RDoc::TopLevel.find_class_or_module rhs_name
else
container.find_module_named rhs_name
end
container.add_module_alias mod, name if mod
get_tk # TkNL
break
end
when TkNL then
if nest <= 0 &&
(@scanner.lex_state == EXPR_END || !@scanner.continue) then
unget_tk tk
break
end
when TkCOLON2, TkCOLON3 then
rhs_name << '::'
when nil then
break
end
tk = get_tk
end
res = get_tkread.tr("\n", " ").strip
res = "" if res == ";"
con = RDoc::Constant.new name, res, comment
read_documentation_modifiers con, RDoc::CONSTANT_MODIFIERS
@stats.add_constant con
container.add_constant con if con.document_self
end
##
# Generates an RDoc::Method or RDoc::Attr from +comment+ by looking for
# :method: or :attr: directives in +comment+.
def parse_comment(container, tk, comment)
line_no = tk.line_no
column = tk.char_no
singleton = !!comment.sub!(/(^# +:?)(singleton-)(method:)/, '\1\3')
# REFACTOR
if comment.sub!(/^# +:?method: *(\S*).*?\n/i, '') then
name = $1 unless $1.empty?
meth = RDoc::GhostMethod.new get_tkread, name
meth.singleton = singleton
meth.start_collecting_tokens
indent = TkSPACE.new nil, 1, 1
indent.set_text " " * column
position_comment = TkCOMMENT.new nil, line_no, 1
position_comment.set_text "# File #{@top_level.absolute_name}, line #{line_no}"
meth.add_tokens [position_comment, NEWLINE_TOKEN, indent]
meth.params = ''
extract_call_seq comment, meth
return unless meth.name
container.add_method meth if meth.document_self
meth.comment = comment
@stats.add_method meth
elsif comment.sub!(/# +:?(attr(_reader|_writer|_accessor)?:) *(\S*).*?\n/i, '') then
rw = case $1
when 'attr_reader' then 'R'
when 'attr_writer' then 'W'
else 'RW'
end
name = $3 unless $3.empty?
att = RDoc::Attr.new get_tkread, name, rw, comment
container.add_attribute att
@stats.add_method att
end
end
def parse_include(context, comment)
loop do
skip_tkspace_comment
name = get_constant_with_optional_parens
context.add_include RDoc::Include.new(name, comment) unless name.empty?
return unless TkCOMMA === peek_tk
get_tk
end
end
##
# Parses a meta-programmed attribute and creates an RDoc::Attr.
#
# To create foo and bar attributes on class C with comment "My attributes":
#
# class C
#
# ##
# # :attr:
# #
# # My attributes
#
# my_attr :foo, :bar
#
# end
#
# To create a foo attribute on class C with comment "My attribute":
#
# class C
#
# ##
# # :attr: foo
# #
# # My attribute
#
# my_attr :foo, :bar
#
# end
def parse_meta_attr(context, single, tk, comment)
args = parse_symbol_arg
read = get_tkread
rw = "?"
# If nodoc is given, don't document any of them
tmp = RDoc::CodeObject.new
read_documentation_modifiers tmp, RDoc::ATTR_MODIFIERS
return unless tmp.document_self
if comment.sub!(/^# +:?(attr(_reader|_writer|_accessor)?): *(\S*).*?\n/i, '') then
rw = case $1
when 'attr_reader' then 'R'
when 'attr_writer' then 'W'
else 'RW'
end
name = $3 unless $3.empty?
end
if name then
att = RDoc::Attr.new get_tkread, name, rw, comment
context.add_attribute att
else
args.each do |attr_name|
att = RDoc::Attr.new get_tkread, attr_name, rw, comment
context.add_attribute att
end
end
end
##
# Parses a meta-programmed method
def parse_meta_method(container, single, tk, comment)
line_no = tk.line_no
column = tk.char_no
start_collecting_tokens
add_token tk
add_token_listener self
skip_tkspace false
singleton = !!comment.sub!(/(^# +:?)(singleton-)(method:)/, '\1\3')
if comment.sub!(/^# +:?method: *(\S*).*?\n/i, '') then
name = $1 unless $1.empty?
end
if name.nil? then
name_t = get_tk
case name_t
when TkSYMBOL then
name = name_t.text[1..-1]
when TkSTRING then
name = name_t.value[1..-2]
when TkASSIGN then # ignore
remove_token_listener self
return
else
warn "unknown name token #{name_t.inspect} for meta-method '#{tk.name}'"
name = 'unknown'
end
end
meth = RDoc::MetaMethod.new get_tkread, name
meth.singleton = singleton
remove_token_listener self
meth.start_collecting_tokens
indent = TkSPACE.new nil, 1, 1
indent.set_text " " * column
position_comment = TkCOMMENT.new nil, line_no, 1
position_comment.value = "# File #{@top_level.absolute_name}, line #{line_no}"
meth.add_tokens [position_comment, NEWLINE_TOKEN, indent]
meth.add_tokens @token_stream
token_listener meth do
meth.params = ''
extract_call_seq comment, meth
container.add_method meth if meth.document_self
last_tk = tk
while tk = get_tk do
case tk
when TkSEMICOLON then
break
when TkNL then
break unless last_tk and TkCOMMA === last_tk
when TkSPACE then
# expression continues
else
last_tk = tk
end
end
end
meth.comment = comment
@stats.add_method meth
end
##
# Parses a normal method defined by +def+
def parse_method(container, single, tk, comment)
added_container = nil
meth = nil
name = nil
line_no = tk.line_no
column = tk.char_no
start_collecting_tokens
add_token tk
token_listener self do
@scanner.instance_eval do @lex_state = EXPR_FNAME end
skip_tkspace false
name_t = get_tk
back_tk = skip_tkspace
meth = nil
added_container = false
dot = get_tk
if TkDOT === dot or TkCOLON2 === dot then
@scanner.instance_eval do @lex_state = EXPR_FNAME end
skip_tkspace
name_t2 = get_tk
case name_t
when TkSELF, TkMOD then
name = name_t2.name
when TkCONSTANT then
name = name_t2.name
prev_container = container
container = container.find_module_named(name_t.name)
unless container then
added_container = true
obj = name_t.name.split("::").inject(Object) do |state, item|
state.const_get(item)
end rescue nil
type = obj.class == Class ? RDoc::NormalClass : RDoc::NormalModule
unless [Class, Module].include?(obj.class) then
warn("Couldn't find #{name_t.name}. Assuming it's a module")
end
if type == RDoc::NormalClass then
sclass = obj.superclass ? obj.superclass.name : nil
container = prev_container.add_class type, name_t.name, sclass
else
container = prev_container.add_module type, name_t.name
end
container.record_location @top_level
end
when TkIDENTIFIER, TkIVAR then
dummy = RDoc::Context.new
dummy.parent = container
skip_method dummy
return
else
warn "unexpected method name token #{name_t.inspect}"
# break
skip_method container
return
end
meth = RDoc::AnyMethod.new(get_tkread, name)
meth.singleton = true
else
unget_tk dot
back_tk.reverse_each do |token|
unget_tk token
end
name = case name_t
when TkSTAR, TkAMPER then
name_t.text
else
unless name_t.respond_to? :name then
warn "expected method name token, . or ::, got #{name_t.inspect}"
skip_method container
return
end
name_t.name
end
meth = RDoc::AnyMethod.new get_tkread, name
meth.singleton = (single == SINGLE)
end
end
meth.start_collecting_tokens
indent = TkSPACE.new nil, 1, 1
indent.set_text " " * column
token = TkCOMMENT.new nil, line_no, 1
token.set_text "# File #{@top_level.absolute_name}, line #{line_no}"
meth.add_tokens [token, NEWLINE_TOKEN, indent]
meth.add_tokens @token_stream
token_listener meth do
@scanner.instance_eval do @continue = false end
parse_method_parameters meth
if meth.document_self then
container.add_method meth
elsif added_container then
container.document_self = false
end
# Having now read the method parameters and documentation modifiers, we
# now know whether we have to rename #initialize to ::new
if name == "initialize" && !meth.singleton then
if meth.dont_rename_initialize then
meth.visibility = :protected
else
meth.singleton = true
meth.name = "new"
meth.visibility = :public
end
end
parse_statements container, single, meth
end
extract_call_seq comment, meth
meth.comment = comment
@stats.add_method meth
end
def parse_method_or_yield_parameters(method = nil,
modifiers = RDoc::METHOD_MODIFIERS)
skip_tkspace false
tk = get_tk
# Little hack going on here. In the statement
# f = 2*(1+yield)
# We see the RPAREN as the next token, so we need
# to exit early. This still won't catch all cases
# (such as "a = yield + 1"
end_token = case tk
when TkLPAREN, TkfLPAREN
TkRPAREN
when TkRPAREN
return ""
else
TkNL
end
nest = 0
loop do
case tk
when TkSEMICOLON then
break
when TkLBRACE then
nest += 1
when TkRBRACE then
# we might have a.each {|i| yield i }
unget_tk(tk) if nest.zero?
nest -= 1
break if nest <= 0
when TkLPAREN, TkfLPAREN then
nest += 1
when end_token then
if end_token == TkRPAREN
nest -= 1
break if @scanner.lex_state == EXPR_END and nest <= 0
else
break unless @scanner.continue
end
when method && method.block_params.nil? && TkCOMMENT then
unget_tk tk
read_documentation_modifiers method, modifiers
@read.pop
when TkCOMMENT then
@read.pop
when nil then
break
end
tk = get_tk
end
res = get_tkread.gsub(/\s+/, ' ').strip
res = '' if res == ';'
res
end
##
# Capture the method's parameters. Along the way, look for a comment
# containing:
#
# # yields: ....
#
# and add this as the block_params for the method
def parse_method_parameters(method)
res = parse_method_or_yield_parameters method
res = "(#{res})" unless res =~ /\A\(/
method.params = res unless method.params
if method.block_params.nil? then
skip_tkspace false
read_documentation_modifiers method, RDoc::METHOD_MODIFIERS
end
end
def parse_module(container, single, tk, comment)
container, name_t = get_class_or_module container
name = name_t.name
mod = container.add_module RDoc::NormalModule, name
mod.record_location @top_level
read_documentation_modifiers mod, RDoc::CLASS_MODIFIERS
parse_statements(mod)
mod.comment = comment
@stats.add_module mod
end
def parse_require(context, comment)
skip_tkspace_comment
tk = get_tk
if TkLPAREN === tk then
skip_tkspace_comment
tk = get_tk
end
name = tk.text if TkSTRING === tk
if name then
context.add_require RDoc::Require.new(name, comment)
else
unget_tk tk
end
end
##
# The core of the ruby parser.
def parse_statements(container, single = NORMAL, current_method = nil,
comment = '')
nest = 1
save_visibility = container.visibility
non_comment_seen = true
while tk = get_tk do
keep_comment = false
non_comment_seen = true unless TkCOMMENT === tk
case tk
when TkNL then
skip_tkspace
tk = get_tk
if TkCOMMENT === tk then
if non_comment_seen then
# Look for RDoc in a comment about to be thrown away
parse_comment container, tk, comment unless comment.empty?
comment = ''
non_comment_seen = false
end
while TkCOMMENT === tk do
comment << tk.text << "\n"
tk = get_tk # this is the newline
skip_tkspace false # leading spaces
tk = get_tk
end
unless comment.empty? then
look_for_directives_in container, comment
if container.done_documenting then
container.ongoing_visibility = save_visibility
end
end
keep_comment = true
else
non_comment_seen = true
end
unget_tk tk
keep_comment = true
when TkCLASS then
if container.document_children then
parse_class container, single, tk, comment
else
nest += 1
end
when TkMODULE then
if container.document_children then
parse_module container, single, tk, comment
else
nest += 1
end
when TkDEF then
if container.document_self then
parse_method container, single, tk, comment
else
nest += 1
end
when TkCONSTANT then
if container.document_self then
parse_constant container, tk, comment
end
when TkALIAS then
if container.document_self and not current_method then
parse_alias container, single, tk, comment
end
when TkYIELD then
if current_method.nil? then
warn "Warning: yield outside of method" if container.document_self
else
parse_yield container, single, tk, current_method
end
# Until and While can have a 'do', which shouldn't increase the nesting.
# We can't solve the general case, but we can handle most occurrences by
# ignoring a do at the end of a line.
when TkUNTIL, TkWHILE then
nest += 1
skip_optional_do_after_expression
# 'for' is trickier
when TkFOR then
nest += 1
skip_for_variable
skip_optional_do_after_expression
when TkCASE, TkDO, TkIF, TkUNLESS, TkBEGIN then
nest += 1
when TkIDENTIFIER then
if nest == 1 and current_method.nil? then
case tk.name
when 'private', 'protected', 'public', 'private_class_method',
'public_class_method', 'module_function' then
parse_visibility container, single, tk
keep_comment = true
when 'attr' then
parse_attr container, single, tk, comment
when /^attr_(reader|writer|accessor)$/ then
parse_attr_accessor container, single, tk, comment
when 'alias_method' then
parse_alias container, single, tk, comment if
container.document_self
when 'require', 'include' then
# ignore
else
if container.document_self and comment =~ /\A#\#$/ then
case comment
when /^# +:?attr(_reader|_writer|_accessor)?:/ then
parse_meta_attr container, single, tk, comment
else
parse_meta_method container, single, tk, comment
end
end
end
end
case tk.name
when "require" then
parse_require container, comment
when "include" then
parse_include container, comment
end
when TkEND then
nest -= 1
if nest == 0 then
read_documentation_modifiers container, RDoc::CLASS_MODIFIERS
container.ongoing_visibility = save_visibility
parse_comment container, tk, comment unless comment.empty?
return
end
end
comment = '' unless keep_comment
begin
get_tkread
skip_tkspace false
end while peek_tk == TkNL
end
end
def parse_symbol_arg(no = nil)
args = []
skip_tkspace_comment
case tk = get_tk
when TkLPAREN
loop do
skip_tkspace_comment
if tk1 = parse_symbol_in_arg
args.push tk1
break if no and args.size >= no
end
skip_tkspace_comment
case tk2 = get_tk
when TkRPAREN
break
when TkCOMMA
else
warn("unexpected token: '#{tk2.inspect}'") if $DEBUG_RDOC
break
end
end
else
unget_tk tk
if tk = parse_symbol_in_arg
args.push tk
return args if no and args.size >= no
end
loop do
skip_tkspace false
tk1 = get_tk
unless TkCOMMA === tk1 then
unget_tk tk1
break
end
skip_tkspace_comment
if tk = parse_symbol_in_arg
args.push tk
break if no and args.size >= no
end
end
end
args
end
def parse_symbol_in_arg
case tk = get_tk
when TkSYMBOL
tk.text.sub(/^:/, '')
when TkSTRING
eval @read[-1]
else
warn("Expected symbol or string, got #{tk.inspect}") if $DEBUG_RDOC
nil
end
end
def parse_top_level_statements(container)
comment = collect_first_comment
look_for_directives_in(container, comment)
container.comment = comment unless comment.empty?
parse_statements container, NORMAL, nil, comment
end
def parse_visibility(container, single, tk)
singleton = (single == SINGLE)
vis_type = tk.name
vis = case vis_type
when 'private' then :private
when 'protected' then :protected
when 'public' then :public
when 'private_class_method' then
singleton = true
:private
when 'public_class_method' then
singleton = true
:public
when 'module_function' then
singleton = true
:public
else
raise RDoc::Error, "Invalid visibility: #{tk.name}"
end
skip_tkspace_comment false
case peek_tk
# Ryan Davis suggested the extension to ignore modifiers, because he
# often writes
#
# protected unless $TESTING
#
when TkNL, TkUNLESS_MOD, TkIF_MOD, TkSEMICOLON then
container.ongoing_visibility = vis
else
if vis_type == 'module_function' then
args = parse_symbol_arg
container.set_visibility_for args, :private, false
module_functions = []
container.methods_matching args do |m|
s_m = m.dup
s_m.singleton = true if RDoc::AnyMethod === s_m
s_m.visibility = :public
module_functions << s_m
end
module_functions.each do |s_m|
case s_m
when RDoc::AnyMethod then
container.add_method s_m
when RDoc::Attr then
container.add_attribute s_m
end
end
else
args = parse_symbol_arg
container.set_visibility_for args, vis, singleton
end
end
end
def parse_yield_parameters
parse_method_or_yield_parameters
end
def parse_yield(context, single, tk, method)
if method.block_params.nil?
get_tkread
@scanner.instance_eval{@continue = false}
method.block_params = parse_yield_parameters
end
end
##
# Directives are modifier comments that can appear after class, module, or
# method names. For example:
#
# def fred # :yields: a, b
#
# or:
#
# class MyClass # :nodoc:
#
# We return the directive name and any parameters as a two element array
def read_directive(allowed)
tk = get_tk
result = nil
if TkCOMMENT === tk then
if tk.text =~ /\s*:?(\w+):\s*(.*)/ then
directive = $1.downcase
if allowed.include? directive then
result = [directive, $2]
end
end
else
unget_tk tk
end
result
end
def read_documentation_modifiers(context, allow)
dir = read_directive(allow)
case dir[0]
when "notnew", "not_new", "not-new" then
context.dont_rename_initialize = true
when "nodoc" then
context.document_self = false
if dir[1].downcase == "all"
context.document_children = false
end
when "doc" then
context.document_self = true
context.force_documentation = true
when "yield", "yields" then
unless context.params.nil?
context.params.sub!(/(,|)\s*&\w+/,'') # remove parameter &proc
end
context.block_params = dir[1]
when "arg", "args" then
context.params = dir[1]
end if dir
end
def remove_private_comments(comment)
comment.gsub!(/^#--\n.*?^#\+\+/m, '')
comment.sub!(/^#--\n.*/m, '')
end
def scan
reset
catch :eof do
catch :enddoc do
begin
parse_top_level_statements @top_level
rescue StandardError => e
bytes = ''
20.times do @scanner.ungetc end
count = 0
60.times do |i|
count = i
byte = @scanner.getc
break unless byte
bytes << byte
end
count -= 20
count.times do @scanner.ungetc end
$stderr.puts <<-EOF
#{self.class} failure around line #{@scanner.line_no} of
#{@file_name}
EOF
unless bytes.empty? then
$stderr.puts
$stderr.puts bytes.inspect
end
raise e
end
end
end
@top_level
end
##
# while, until, and for have an optional do
def skip_optional_do_after_expression
skip_tkspace false
tk = get_tk
case tk
when TkLPAREN, TkfLPAREN
end_token = TkRPAREN
else
end_token = TkNL
end
nest = 0
@scanner.instance_eval{@continue = false}
loop do
case tk
when TkSEMICOLON
break
when TkLPAREN, TkfLPAREN
nest += 1
when TkDO
break if nest.zero?
when end_token
if end_token == TkRPAREN
nest -= 1
break if @scanner.lex_state == EXPR_END and nest.zero?
else
break unless @scanner.continue
end
when nil then
break
end
tk = get_tk
end
skip_tkspace false
get_tk if TkDO === peek_tk
end
##
# skip the var [in] part of a 'for' statement
def skip_for_variable
skip_tkspace false
tk = get_tk
skip_tkspace false
tk = get_tk
unget_tk(tk) unless TkIN === tk
end
def skip_method container
meth = RDoc::AnyMethod.new "", "anon"
parse_method_parameters meth
parse_statements container, false, meth
end
##
# Skip spaces until a comment is found
def skip_tkspace_comment(skip_nl = true)
loop do
skip_tkspace skip_nl
return unless TkCOMMENT === peek_tk
get_tk
end
end
def warn(msg)
return if @options.quiet
msg = make_message msg
$stderr.puts msg
end
end
|