1 files changed, 861 insertions, 0 deletions
diff --git a/lib/asn1kit/parse.ry b/lib/asn1kit/parse.ry
new file mode 100644
index 0000000..d5ffd76
--- /dev/null
+++ b/lib/asn1kit/parse.ry
@@ -0,0 +1,861 @@
+class ASN1Kit::Parser
+prechigh
+  nonassoc EXCEPT
+  left "^" INTERSECTION
+  left "|" UNION
+preclow
+
+token
+  Cstring Hstring Bstring Number Negative_number Realnumber
+  Identifier Typereference Objectclassreference
+  Unparsed
+
+  # For parsing UnparsedValueNode
+  __CompoundBitStringValue __ObjectIdentifierValue __SequenceValue
+  __SequenceOfValue
+
+  # Reserved words
+  ABSENT ENCODED INTERS ECTION SEQUENCE ABSTRACT_SYNTAX ENCODING_CONTROL
+  ISO646String SET ALL END MAX SETTINGS APPLICATION ENUMERATED MIN SIZE
+  AUTOMATIC EXCEPT MINUS_INFINITY STRING BEGIN EXPLICIT NOT_A_NUMBER
+  SYNTAX BIT EXPORTS NULL T61String BMPString EXTENSIBILITY NumericString
+  TAGS BOOLEAN EXTERNAL OBJECT TeletexString BY FALSE ObjectDescriptor
+  TIME CHARACTER FROM OCTET TIME_OF_DAY CHOICE GeneralizedTime OF TRUE
+  CLASS GeneralString OID_IRI COMPONENT GraphicString
+  OPTIONAL UNION INTERSECTION COMPONENTS IA5String PATTERN UNIQUE CONSTRAINED
+  IDENTIFIER PDV UNIVERSAL CONTAINING IMPLICIT PLUS_INFINITY
+  UniversalString DATE IMPLIED PRESENT UTCTime DATE_TIME IMPORTS
+  PrintableString UTF8String DEFAULT INCLUDES PRIVATE VideotexString
+  DEFINITIONS INSTANCE REAL VisibleString DURATION INSTRUCTIONS
+  RELATIVE_OID WITH EMBEDDED INTEGER RELATIVE_OID_IRI
+
+start root
+
+rule
+  # X.680 13. Module definition
+  moduleDefinition
+    : modulereference definitiveIdentification
+      DEFINITIONS
+      tagDefault
+      extensionDefault
+      "::="
+      BEGIN
+      moduleBody
+      END
+      { result = D::ModuleDefinitionNode.new(val[0], val[1], val[3], val[4], val[7]) }
+    ;
+
+  definitiveIdentification
+    : # empty
+    # DefinitiveOID
+    | "{" definitiveObjIdComponentList "}"
+      { result = D::ObjectIdentifierValueNode.new(val[1]) }
+    # DefinitiveOIDAndIRI form is not supported
+    ;
+
+  definitiveObjIdComponentList
+    : definitiveObjIdComponent
+      { result = [val[0]] }
+    | definitiveObjIdComponentList definitiveObjIdComponent
+      { result << val[1] }
+    ;
+
+  definitiveObjIdComponent # FIXME: type?
+    : Identifier # NameForm
+      { result = [val[0], nil] }
+    | Number # DefinitiveNumberForm
+      { result = [nil, val[0]] }
+    | Identifier "(" Number ")" # DefinitiveNameAndNumberForm
+      { result = [val[0], val[2]] }
+    ;
+
+  tagDefault
+    : # empty
+      { result = :EXPLICIT }
+    | EXPLICIT TAGS
+      { result = :EXPLICIT }
+    | IMPLICIT TAGS
+      { result = :IMPLICIT }
+    | AUTOMATIC TAGS
+      { result = :AUTOMATIC }
+    ;
+
+  extensionDefault
+    : # empty
+      { result = false }
+    | EXTENSIBILITY IMPLIED
+      { result = true }
+    ;
+
+  moduleBody
+    : # empty
+      { result = [] }
+    | exports imports assignmentList
+      { result = val }
+    ;
+
+  exports
+    : # empty
+    | EXPORTS ";"
+    | EXPORTS symbolList ";"
+    | EXPORTS ALL ";"
+    ;
+
+  imports
+    : # empty
+      { result = [] }
+    | IMPORTS ";"
+      { result = [] }
+    | IMPORTS symbolsFromModuleList ";"
+      { raise ParseError, "Imports not supported" }
+    ;
+
+  symbolsFromModuleList
+    : symbolsFromModule
+    | symbolsFromModuleList symbolsFromModule
+    ;
+
+  symbolsFromModule
+    : symbolList FROM modulereference assignedIdentifier;
+    ;
+
+  assignedIdentifier
+    : # empty
+    | objectIdentifierValue
+    # FIXME: Resolve shift/reduce conflict
+    # | definedValue
+    ;
+
+  symbolList
+    : reference
+    | symbolList "," reference
+    ;
+
+  reference
+    : typereference # including objectclassreference, objectsetreference
+    | Identifier # valuereference, objectreference
+    ;
+
+  assignmentList
+    : assignment
+      { result = [val[0]] }
+    | assignmentList assignment
+      { result << val[1] }
+    ;
+
+  # 14. Referencing type and value definitions
+  assignment
+    # TypeAssignment
+    : typereference "::=" type
+      { result = [val[0], val[2]] }
+    # ValueAssignment
+    | Identifier type "::=" value
+      { result = [val[0], val[1], val[3]] }
+    ;
+
+  definedType
+    : typereference
+      { result = D::TypereferenceNode.new(val[0]) }
+    | modulereference "." typereference
+      { raise ParseError, "ExternalTypeReference not supported" }
+    ;
+
+  definedValue
+    : Identifier
+      # May be INTEGER/ENUMERATED identifier
+      { result = D::ValuereferenceNode.new(val[0]) }
+    | modulereference "." Identifier
+      { raise ParseError, "ExternalValueReference not supported" }
+    ;
+
+  # 17. Definition of types and values
+  type
+    : builtinType
+    | referencedType
+    | constrainedType
+    ;
+
+  builtinType
+    # 18. BooleanType
+    : BOOLEAN
+      { result = D::SimpleTypeNode.new(:BOOLEAN) }
+    # 19. IntegerType
+    | INTEGER
+      { result = D::SimpleTypeNode.new(:INTEGER) }
+    | INTEGER '{' namedNumberList '}'
+      { result = D::SimpleTypeNode.new(:INTEGER, val[2]) }
+    # 20. EnumeratedType
+    | ENUMERATED "{" enumerations "}"
+      { result = D::SimpleTypeNode.new(:ENUMERATED, val[2]) }
+    # 21. RealType
+    | REAL
+      { result = D::SimpleTypeNode.new(:REAL) }
+    # 22. BitStringType
+    | BIT STRING
+      { result = D::SimpleTypeNode.new(:BIT_STRING) }
+    | BIT STRING "{" namedBitList "}"
+      { result = D::SimpleTypeNode.new(:BIT_STRING, val[3]) }
+    # 23. OctetStringType
+    | OCTET STRING
+      { result = D::SimpleTypeNode.new(:OCTET_STRING) }
+    # 24. NullType
+    | NULL
+      { result = D::SimpleTypeNode.new(:NULL) }
+    # 25. SequenceType
+    | SEQUENCE "{" "}"
+      { result = D::SequenceTypeNode.new([]) }
+    | SEQUENCE "{" componentTypeLists "}"
+      { result = D::SequenceTypeNode.new(val[2]) }
+    # 26. SequenceOfType
+    | SEQUENCE OF type
+      { result = D::SequenceOfTypeNode.new(val[2]) }
+    | SEQUENCE OF namedType
+      { result = D::SequenceOfTypeNode.new(val[2].type, type_name: val[2].name) }
+    # 27. SetType
+    | SET "{" "}"
+      { result = D::SetTypeNode.new([]) }
+    | SET "{" componentTypeLists "}"
+      { result = D::SetTypeNode.new(val[2]) }
+    # 28. SetOfType
+    | SET OF type
+      { result = D::SetOfTypeNode.new(val[2]) }
+    | SET OF namedType
+      { result = D::SetOfTypeNode.new(val[2].type, type_name: val[2].name) }
+    # 29. ChoiceType
+    | CHOICE "{" alternativeTypeList "}"
+      { result = D::SimpleTypeNode.new(:CHOICE, val[2]) }
+    # 31. PrefixedType
+    | prefixedType
+    # 32. ObjectIdentifierType
+    | OBJECT IDENTIFIER
+      { result = D::SimpleTypeNode.new(:OBJECT_IDENTIFIER) }
+    # 33. RelativeOIDType
+    | RELATIVE_OID
+      { result = D::SimpleTypeNode.new(:RELATIVE_OID) }
+    # 34. IRIType
+    | OID_IRI
+      { raise ParseError, "OID-IRI type not supported" }
+    # 35. RelativeIRIType
+    | RELATIVE_OID_IRI
+      { raise ParseError, "RELATIVE-OID-IRI type not supported" }
+    # 36. EmbeddedPDVType
+    | EMBEDDED PDV
+      { raise ParseError, "EMBEDDED PDV type not supported" }
+    # 37. ExternalType
+    | EXTERNAL
+      { raise ParseError, "EXTERNAL type not supported" }
+    # 38.1.1. TimeType
+    | TIME
+      { raise ParseError, "TIME type not supported" }
+    # 38.4.1. DateType
+    | DATE
+      { raise ParseError, "DATE type not supported" }
+    # 38.4.2. TimeOfDayType
+    | TIME_OF_DAY
+      { raise ParseError, "TIME-OF-DAY type not supported" }
+    # 38.4.3. DateTimeType
+    | DATE_TIME
+      { raise ParseError, "DATE-TIME type not supported" }
+    # 38.4.4. DurationType
+    | DURATION
+      { raise ParseError, "DURATION type not supported" }
+    # 40. CharacterStringType
+    | characterStringType
+    # X.681 Annex C. InstanceOfType
+    # X.681 14.1. ObjectClassFieldType
+    ;
+
+  referencedType
+    : definedType
+    # UsefulType; actually typeidentifier, but only these three are possible
+    | GeneralizedTime
+      { result = D::SimpleTypeNode.new(:GeneralizedTime) }
+    | UTCTime
+      { result = D::SimpleTypeNode.new(:UTCTime) }
+    | ObjectDescriptor
+      { result = D::SimpleTypeNode.new(:ObjectDescriptor) }
+    ;
+
+  namedType
+    : Identifier type
+      { result = D::NamedTypeNode.new(val[0], val[1]) }
+    ;
+
+  value
+    : builtinValue
+    # ReferencedValue and identifier alternative of IntegerValue
+    # and EnumeratedValue
+    | referencedValue
+    ;
+
+  builtinValue
+    : TRUE
+      { result = D::SimpleValueNode.new(:boolean, true) }
+    | FALSE
+      { result = D::SimpleValueNode.new(:boolean, false) }
+    | signedNumber
+      { result = D::SimpleValueNode.new(:integer, val[0]) }
+    | realValue
+    | Bstring
+      { result = D::SimpleValueNode.new(:bstring, val[0]) }
+    | Hstring
+      { result = D::SimpleValueNode.new(:hstring, val[0]) }
+    | NULL
+      { result = D::SimpleValueNode.new(:null, nil) }
+    # BitStringValue, {Sequence,Set}{Of,}Value, ObjectIdentifierValue,
+    # RelativeOIDValue, and RestrictedCharacterStringValue.
+    | unparsedValue
+    # IRIValue, RelativeIRIValue, and TimeValue are not supported
+    ;
+
+  # "{" something "}" in value is ambiguous. Let's parse after determining the
+  # type.
+  unparsedValue
+    : "{" { @lex_state << :unparsed_value } unparsed # "}"
+      { result = D::UnparsedValueNode.new("{" << val[2]) }
+    ;
+
+  unparsed
+    : Unparsed
+    | unparsed Unparsed
+      { result << val[1] }
+    ;
+
+  referencedValue
+    : definedValue
+    ;
+
+  namedNumberList
+    : namedNumber {
+      result = [val[0]] }
+    | namedNumberList ',' namedNumber
+      { result << val[2] }
+    ;
+
+  namedNumber
+    : Identifier '(' signedNumber ')' {
+        result = [val[0], val[2]] }
+    | Identifier '(' definedValue ')' {
+        result = [val[0], val[2]] }
+    ;
+
+  signedNumber
+    : Number
+    | Negative_number
+    ;
+
+  # X.680 20 Notation for the enumerated type
+  enumerations
+    : enumeration
+      { result = [val[0], nil] }
+    | enumeration "," "..." exceptionSpec
+      { result = [val[0], []] }
+    | enumeration "," "..." exceptionSpec "," enumeration
+      { result = [val[0], val[5]] }
+    ;
+
+  enumeration
+    : enumerationItem
+      { result = [val[0]] }
+    | enumeration "," enumerationItem
+      { result << val[2] }
+    ;
+
+  enumerationItem
+    : Identifier
+      { result = [val[0], nil] }
+    | namedNumber
+    ;
+
+  # 21. Notation for the real type
+  realValue
+    # NumericRealValue - SequenceValue form is omitted
+    : Realnumber # Includes '"-" realnumber' form
+      { result = D::SimpleValueNode.new(:real, val[0]) }
+    # SpecialRealValue
+    | PLUS_INFINITY
+      { result = D::SimpleValueNode.new(:real, :PLUS_INFINITY) }
+    | MINUS_INFINITY
+      { result = D::SimpleValueNode.new(:real, :MINUS_INFINITY) }
+    | NOT_A_NUMBER
+      { result = D::SimpleValueNode.new(:real, :NOT_A_NUMBER) }
+    ;
+
+
+  # X.680 22 Notation for the bitstring type
+  namedBit
+    : Identifier "(" Number ")"
+      { result = [val[0], val[2]] }
+    | Identifier "(" definedValue ")"
+      { result = [val[0], val[2]] }
+    ;
+
+  namedBitList
+    : namedBit
+      { result = [val[0]] }
+    | namedBitList "," namedBit
+      { result << val[2] }
+    ;
+
+  # X.680 25 Notation for sequence types
+  extensionAndException
+    | "..." exceptionSpec
+      { raise ParseError, "ExceptionSpec in ComponentTypeLists not supported" if val[1] }
+    ;
+
+  componentTypeLists
+    : componentTypeList
+      { result = [val[0]] }
+    | componentTypeList "," extensionAndException extensionAdditions
+      { result = [val[0], :extension, val[3]] }
+    | componentTypeList "," extensionAndException extensionAdditions "," "..."
+      { result = [val[0], :extension, val[3], :extension] }
+    | componentTypeList "," extensionAndException extensionAdditions "," "..." "," componentTypeList
+      { result = [val[0], :extension, val[3], :extension, val[7]] }
+    | extensionAndException extensionAdditions
+      { result = [:extension, val[1]] }
+    | extensionAndException extensionAdditions "," "..."
+      { result = [:extension, val[1], :extension] }
+    | extensionAndException extensionAdditions "," "..." "," componentTypeList
+      { result = [:extension, val[1], :extension, val[5]] }
+    ;
+
+  extensionAdditions
+    : # empty
+      { result = [] }
+    | "," extensionAdditionList
+      { result = val[1] }
+    ;
+
+  extensionAdditionList
+    : extensionAddition
+      { result = [val[0]] }
+    | extensionAdditionList "," extensionAddition
+      { result << val[2] }
+    ;
+
+  extensionAddition
+    : componentType
+    | extensionAdditionGroup
+      { raise ParseError, "ExtensionAdditionGroup not supported" }
+    ;
+
+  extensionAdditionGroup
+    : "[[" componentTypeLists "]]"
+    | "[[" Number ":" componentTypeLists "]]"
+    ;
+
+  componentTypeList
+    : componentType
+      { result = [val[0]] }
+    | componentTypeList ',' componentType
+      { result << val[2] }
+    ;
+
+  componentType
+    : Identifier type
+      { result = D::ComponentTypeNode.new(val[0], val[1]) }
+    | Identifier type DEFAULT value
+      { result = D::ComponentTypeNode.new(val[0], val[1], default: val[3]) }
+    | Identifier type OPTIONAL
+      { result = D::ComponentTypeNode.new(val[0], val[1], optional: true) }
+    ;
+
+  alternativeTypeList
+    : namedType
+      { result = [val[0]] }
+    | alternativeTypeList "," namedType
+      { result << val[2] }
+    ;
+
+
+
+  prefixedType
+    : taggedType
+    ;
+
+  taggedType
+    : "[" class classNumber "]" type
+      { result = D::TaggedTypeNode.new(val[4], nil, val[1], val[2]) }
+    | "[" class classNumber "]" IMPLICIT type
+      { result = D::TaggedTypeNode.new(val[5], :implicit, val[1], val[2]) }
+    | "[" class classNumber "]" EXPLICIT type
+      { result = D::TaggedTypeNode.new(val[5], :explicit, val[1], val[2]) }
+    ;
+
+  classNumber
+    : Number
+    | definedValue
+    ;
+
+  class
+    : # empty
+      { result = :CONTEXT_SPECIFIC }
+    | UNIVERSAL
+      { result = :UNIVERSAL }
+    | APPLICATION
+      { result = :APPLICATION }
+    | PRIVATE
+      { result = :PRIVATE }
+    ;
+
+
+  objectIdentifierValue
+    # Includes "{" definedValue objIdComponentsList "}" pattern
+    : "{" objIdComponentsList "}"
+      { result = D::ObjectIdentifierValueNode.new(val[1]) }
+    ;
+
+  objIdComponentsList
+    : objIdComponents
+      { result = [val[0]] }
+    | objIdComponentsList objIdComponents
+      { result << val[1] }
+    ;
+
+  objIdComponents
+    : Number
+      { result = [nil, val[0]] }
+    | Identifier
+      { result = [val[0], nil] }
+    | Identifier "(" Number ")"
+      { result = [val[0], val[2]] }
+    | Identifier "(" Identifier ")"
+      { result = [val[0], val[2]] }
+    ;
+
+
+  characterStringType
+    : restrictedCharacterStringType
+      { result = D::SimpleTypeNode.new(val[0].intern) }
+    # unrestrictedCharacterStringType
+    | CHARACTER STRING
+      { result = D::SimpleTypeNode.new(:CHARACTER_STRING) }
+    ;
+
+  restrictedCharacterStringType
+    : BMPString
+    | GeneralString
+    | GraphicString
+    | IA5String
+    | ISO646String
+    | NumericString
+    | PrintableString
+    | TeletexString
+    | T61String
+    | UniversalString
+    | UTF8String
+    | VideotexString
+    | VisibleString
+    ;
+
+
+
+
+  constrainedType
+    : type constraint
+      { result = val[0].update(constraint: val[1]) }
+    | typeWithConstraint
+    ;
+
+  typeWithConstraint
+    : SET constraint OF type
+      { result = D::SetOfTypeNode.new(val[3]).update(constraint: val[1]) }
+    | SET sizeConstraint OF type
+      { result = D::SetOfTypeNode.new(val[3]).update(constraint: val[1]) }
+    | SEQUENCE constraint OF type
+      { result = D::SequenceOfTypeNode.new(val[3]).update(constraint: val[1]) }
+    | SEQUENCE sizeConstraint OF type
+      { result = D::SequenceOfTypeNode.new(val[3]).update(constraint: val[1]) }
+    | SET constraint OF namedType
+      { result = D::SetOfTypeNode.new(val[3].type, type_name: val[3].name).update(constraint: val[1]) }
+    | SET sizeConstraint OF namedType
+      { result = D::SetOfTypeNode.new(val[3].type, type_name: val[3].name).update(constraint: val[1]) }
+    | SEQUENCE constraint OF namedType
+      { result = D::SequenceOfTypeNode.new(val[3].type, type_name: val[3].name).update(constraint: val[1]) }
+    | SEQUENCE sizeConstraint OF namedType
+      { result = D::SequenceOfTypeNode.new(val[3].type, type_name: val[3].name).update(constraint: val[1]) }
+    ;
+
+  constraint
+    : "(" constraintSpec exceptionSpec ")"
+      { result = val[1] }
+    ;
+
+  constraintSpec
+    # SubtypeConstraint
+    : elementSetSpecs
+    ;
+
+  elementSetSpecs
+    : elementSetSpec
+    | elementSetSpec "," "..."
+    | elementSetSpec "," "..." "," elementSetSpec
+    ;
+
+  elementSetSpec
+    : unions
+    | ALL EXCEPT elements
+    ;
+
+  unions
+    : intersections
+    | unions unionMark intersections
+    ;
+
+  intersections
+    : intersectionElements
+    | intersections intersectionMark intersectionElements
+    ;
+
+  intersectionElements
+    : elements
+    | elements EXCEPT elements
+    ;
+
+  unionMark : "|" | UNION ;
+
+  intersectionMark : "^" | INTERSECTION ;
+
+  elements
+    : subtypeElements
+    | "(" elementSetSpec ")"
+    ;
+
+  subtypeElements
+    # SingleValue
+    : value
+      { result = D::Constraint.new(:SingleValue, val[0]) }
+    # ValueRange
+    | lowerEndpoint ".." upperEndpoint
+      { result = D::ValueRange.new(val[0], val[2]) }
+    # SizeConstraint
+    | sizeConstraint
+    ;
+
+  lowerEndpoint
+    : lowerEndValue
+      { result = [val[0], :inclusive] }
+    | lowerEndValue "<"
+      { result = [val[0], :exclusive] }
+    ;
+
+  upperEndpoint
+    : upperEndValue
+      { result = [val[0], :inclusive] }
+    | "<" upperEndValue
+      { result = [val[1], :exclusive] }
+    ;
+
+  lowerEndValue : value | MIN ;
+  upperEndValue : value | MAX ;
+
+  sizeConstraint
+    : SIZE constraint
+      { result = D::SizeConstraint.new(val[1]) }
+    ;
+
+  # 53
+  exceptionSpec
+    : # empty
+    | "!" exceptionIdentification
+      { raise ParseError, "ExceptionSpec not supported yet" }
+    ;
+
+  exceptionIdentification
+    : signedNumber
+    | definedValue
+    | type ":" value
+    ;
+
+  typereference : Objectclassreference | Typereference ;
+  modulereference : Objectclassreference | Typereference ;
+
+  # Hack for UnparsedValueNode. A valid ASN.1 module never starts with the
+  # character '_' so I think it is safe.... though ugly
+  root
+    : moduleDefinition
+    | __CompoundBitStringValue compoundBitStringValue
+      { result = val[1] }
+    | __ObjectIdentifierValue objectIdentifierValue
+      { result = val[1] }
+    | __SequenceValue sequenceValue
+      { result = val[1] }
+    | __SequenceOfValue sequenceOfValue
+      { result = val[1] }
+    ;
+
+  compoundBitStringValue
+    : "{" "}"
+      { result = [] }
+    | "{" identifierList "}"
+      { result = val[1] }
+    ;
+
+  identifierList
+    : Identifier
+      { result = [val[0]] }
+    | identifierList "," Identifier
+      { result << val[2] }
+    ;
+
+  sequenceValue
+    : "{" "}"
+      { result = [] }
+    | "{" componentValueList "}"
+      { result = val[1] }
+    ;
+
+  componentValueList
+    : namedValue
+      { result = [val[0]] }
+    | componentValueList "," namedValue
+      { result << val[2] }
+    ;
+
+  namedValue
+    : Identifier value
+      { result = val }
+    ;
+
+  sequenceOfValue
+    : "{" "}"
+      { result = [] }
+    | "{" valueList "}"
+      { result = val[1] }
+    # Equivalent to NamedValueList
+    | "{" componentValueList "}"
+      { result = val[1] }
+    ;
+
+  valueList
+    : value
+      { result = [[nil, val[0]]] }
+    | valueList "," value
+      { result << [nil, val[2]] }
+    ;
+
+---- inner
+
+  D = ASN1Kit::Internal::Nodes
+  private_constant :D
+
+  LEXICAL_KEYWORDS = %w{
+    ANY ABSENT ALL APPLICATION AUTOMATIC BEGIN BIT BMPString BOOLEAN BY
+    CHARACTER CHOICE CLASS COMPONENT COMPONENTS CONSTRAINED CONTAINING DATE
+    DATE-TIME DEFAULT DEFINITIONS DURATION EMBEDDED ENCODED ENCODING-CONTROL
+    END ENUMERATED EXCEPT EXPLICIT EXPORTS EXTENSIBILITY EXTERNAL FALSE FROM
+    GeneralizedTime GeneralString GraphicString IA5String IDENTIFIER
+    IMPLICIT IMPLIED IMPORTS INCLUDES INSTANCE INSTRUCTIONS INTEGER
+    INTERSECTION ISO646String MAX MINUS-INFINITY MIN NOT-A-NUMBER NULL
+    NumericString OBJECT ObjectDescriptor OCTET OF OID-IRI OPTIONAL PATTERN
+    PDV PLUS-INFINITY PRESENT PrintableString PRIVATE REAL RELATIVE-OID
+    RELATIVE-OID-IRI SEQUENCE SET SETTINGS SIZE STRING SYNTAX T61String TAGS
+    TeletexString TIME TIME-OF-DAY TRUE UNION UNIQUE UNIVERSAL
+    UniversalString UTCTime UTF8String VideotexString VisibleString WITH
+  }
+
+  WS = "[\n\v\f\r ]"
+
+  def next_token
+    token = nil
+    s = @scanner
+    state = @lex_state
+
+    cstring = nil
+
+    until token or s.eos?
+      @lineno += 1 if s.peek(1) == ?\n
+      case state.last
+      when nil
+        case
+        when s.skip(/--.*?(?:--|$)/)
+        when s.skip(/\/\*/)
+          state << :block_comment
+        when t = s.scan(/__\w+/) # Internal
+          token = [t.intern, t]
+        when t = s.scan(/"[^"]*/)
+          cstring = text[1..-1]
+          state << :cstring
+        when t = s.scan(/'(?:[0-9A-F]|#{WS})+'H/o)
+          token = [:Hstring, t]
+        when t = s.scan(/'(?:[01]|#{WS})+'B/o)
+          token = [:Bstring, t]
+        when t = s.scan(/-[1-9][0-9]*/)
+          token = [:Negative_number, t.to_i]
+        when t = s.scan(/(?:0|[1-9][0-9]*)/)
+          token = [:Number, t.to_i]
+        when t = s.scan(/[-+]?[0-9]+\.?(?:[eE][-+]?)?[0-9]+/)
+          token = [:Realnumber, t.to_i]
+        when t = s.scan(/(?:#{LEXICAL_KEYWORDS.join("|")})/o)
+          token = [t.tr("-", "_").intern, t]
+        when t = s.scan(/[a-z][A-Za-z0-9]*(?:-[A-Za-z0-9]+)*/)
+          token = [:Identifier, t]
+        when t = s.scan(/[A-Z][A-Za-z0-9]*(?:-[A-Za-z0-9]+)*/)
+          token = [:Typereference, t]
+        when t = s.scan(/[A-Z][A-Z0-9]*(?:-[A-Z0-9]+)*/)
+          token = [:Objectclassreference, t]
+        when t = s.scan(/(?:::=|\.\.\.|\.\.|\[\[|\]\])/)
+          token = [t, t]
+        when t = s.scan(/[(){},.;:!|&@\[\]^]/)
+          token = [t, t]
+        when s.skip(/#{WS}+/o)
+        else unreachable
+        end
+      when :block_comment
+        case
+        when s.skip(/[^*\/]+/)
+        when s.skip(/\*\//)
+          state.pop
+        when s.skip(/\/\*/)
+          state << :block_comment
+        when s.skip(/./)
+        else unreachable
+        end
+      when :cstring
+        case
+        when s.skip(/""/)
+          cstring << ?"
+        when t = s.scan(/[^"]+/)
+          cstring << t
+        when s.skip(/"/)
+          state.pop
+          token = [:Cstring, cstring]
+        else unreachable
+        end
+      when :unparsed_value
+        # This state is pushed from parser on demand.
+        case
+        when s.skip(/{/)
+          state << :unparsed_value
+          token = [:Unparsed, "{"]
+        when s.skip(/}/)
+          state.pop
+          token = [:Unparsed, "}"]
+        when t = s.scan(/[^{}]+/)
+          token = [:Unparsed, t]
+        else unreachable
+        end
+      else unreachable
+      end
+    end
+
+    token || [false, "$"]
+  end
+
+  def on_error(a, b, c)
+    p [a, b, c]
+    p @lex_state
+    raise "on error"
+  end
+
+  def parse(str)
+    @scanner = StringScanner.new(str)
+    @lineno = 1
+    @lex_state = [nil]
+    do_parse
+  end
+
+  private def unreachable
+    raise "internal error: unreachable"
+  end