From 0577e85671ba729a092dcdcf0f3d2b7e948ed272 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Thu, 6 Dec 2018 10:09:41 -0500 Subject: [PATCH 1/5] Implement Syntax 0.8 --- .gitattributes | 1 + fluent/syntax/ast.py | 14 +- fluent/syntax/errors.py | 14 +- fluent/syntax/parser.py | 484 ++++++++++------- fluent/syntax/serializer.py | 103 ++-- fluent/syntax/stream.py | 135 ++--- .../call_expression_with_bad_id.ftl | 2 +- .../call_expression_with_wrong_value_type.ftl | 2 +- .../fixtures_behavior/escape_sequences.ftl | 22 +- .../placeable_in_placeable.ftl | 3 +- .../select_expression_without_variants.ftl | 2 +- .../selector_expression_ends_abruptly.ftl | 2 +- tests/syntax/fixtures_behavior/term.ftl | 13 +- .../unclosed_empty_placeable_error.ftl | 2 +- .../variant_expression_as_placeable.ftl | 2 +- .../variant_expression_as_selector.ftl | 2 +- .../fixtures_behavior/variant_lists.ftl | 7 +- .../syntax/fixtures_structure/blank_lines.ftl | 27 + .../fixtures_structure/blank_lines.json | 271 ++++++++++ tests/syntax/fixtures_structure/crlf.json | 2 +- .../fixtures_structure/escape_sequences.ftl | 23 +- .../fixtures_structure/escape_sequences.json | 494 ++++++++++++++--- .../expressions_call_args.json | 12 +- tests/syntax/fixtures_structure/junk.ftl | 23 + tests/syntax/fixtures_structure/junk.json | 238 +++++++++ .../fixtures_structure/leading_dots.json | 21 +- .../fixtures_structure/multiline_pattern.json | 6 +- .../fixtures_structure/placeable_at_eol.json | 4 +- .../fixtures_structure/select_expressions.ftl | 9 + .../select_expressions.json | 72 +++ .../fixtures_structure/sparse-messages.json | 14 +- tests/syntax/fixtures_structure/term.json | 30 +- .../term_with_empty_pattern.json | 4 +- .../fixtures_structure/variant_keys.ftl | 61 +++ .../fixtures_structure/variant_keys.json | 500 ++++++++++++++++++ .../variant_with_empty_pattern.json | 5 +- .../whitespace_leading.json | 2 + .../whitespace_trailing.json | 1 + tests/syntax/test_serializer.py | 12 +- 39 files changed, 2134 insertions(+), 507 deletions(-) create mode 100644 tests/syntax/fixtures_structure/blank_lines.ftl create mode 100644 tests/syntax/fixtures_structure/blank_lines.json create mode 100644 tests/syntax/fixtures_structure/junk.ftl create mode 100644 tests/syntax/fixtures_structure/junk.json create mode 100644 tests/syntax/fixtures_structure/select_expressions.ftl create mode 100644 tests/syntax/fixtures_structure/select_expressions.json create mode 100644 tests/syntax/fixtures_structure/variant_keys.ftl create mode 100644 tests/syntax/fixtures_structure/variant_keys.json diff --git a/.gitattributes b/.gitattributes index fc43c62e..74346c43 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ tests/syntax/fixtures_reference/crlf.ftl eol=crlf +tests/syntax/fixtures_reference/cr.ftl eol=cr tests/syntax/fixtures_structure/crlf.ftl eol=crlf diff --git a/fluent/syntax/ast.py b/fluent/syntax/ast.py index 6d0eff75..4feaa841 100644 --- a/fluent/syntax/ast.py +++ b/fluent/syntax/ast.py @@ -207,8 +207,9 @@ class Expression(SyntaxNode): class StringLiteral(Expression): - def __init__(self, value, **kwargs): + def __init__(self, raw, value, **kwargs): super(StringLiteral, self).__init__(**kwargs) + self.raw = raw self.value = value @@ -236,6 +237,12 @@ def __init__(self, id, **kwargs): self.id = id +class FunctionReference(Expression): + def __init__(self, id, **kwargs): + super(FunctionReference, self).__init__(**kwargs) + self.id = id + + class SelectExpression(Expression): def __init__(self, selector, variants, **kwargs): super(SelectExpression, self).__init__(**kwargs) @@ -324,11 +331,6 @@ def __init__(self, content=None, **kwargs): super(ResourceComment, self).__init__(content, **kwargs) -class Function(Identifier): - def __init__(self, name, **kwargs): - super(Function, self).__init__(name, **kwargs) - - class Junk(SyntaxNode): def __init__(self, content=None, annotations=None, **kwargs): super(Junk, self).__init__(**kwargs) diff --git a/fluent/syntax/errors.py b/fluent/syntax/errors.py index a3d2324b..bdbc747e 100644 --- a/fluent/syntax/errors.py +++ b/fluent/syntax/errors.py @@ -21,12 +21,12 @@ def get_error_message(code, args): msg = 'Expected message "{}" to have a value or attributes' return msg.format(args[0]) if code == 'E0006': - msg = 'Expected term "{}" to have a value' + msg = 'Expected term "-{}" to have a value' return msg.format(args[0]) if code == 'E0007': return 'Keyword cannot end with a whitespace' if code == 'E0008': - return 'The callee has to be a simple, upper-case identifier' + return 'The callee has to be an upper-case identifier or a term' if code == 'E0009': return 'The key has to be a simple identifier' if code == 'E0010': @@ -44,7 +44,7 @@ def get_error_message(code, args): if code == 'E0016': return 'Message references cannot be used as selectors' if code == 'E0017': - return 'Variants cannot be used as selectors' + return 'Terms cannot be used as selectors' if code == 'E0018': return 'Attributes of messages cannot be used as selectors' if code == 'E0019': @@ -55,12 +55,14 @@ def get_error_message(code, args): return 'Positional arguments must not follow named arguments' if code == 'E0022': return 'Named arguments must be unique' - if code == 'E0023': - return 'VariantLists are only allowed inside of other VariantLists.' if code == 'E0024': return 'Cannot access variants of a message.' if code == 'E0025': - return 'Unknown escape sequence: {}'.format(args[0]) + return 'Unknown escape sequence: \\{}.'.format(args[0]) if code == 'E0026': return 'Invalid Unicode escape sequence: {}'.format(args[0]) + if code == 'E0027': + return 'Unbalanced closing brace in TextElement.' + if code == 'E0028': + return 'Expected an inline expression' return code diff --git a/fluent/syntax/parser.py b/fluent/syntax/parser.py index 318a6f05..57ebf8d0 100644 --- a/fluent/syntax/parser.py +++ b/fluent/syntax/parser.py @@ -4,16 +4,21 @@ from .stream import EOF, EOL, FluentParserStream from .errors import ParseError +try: + from __builtin__ import unichr as chr +except ModuleNotFoundError: + pass + def with_span(fn): - def decorated(self, ps, *args): + def decorated(self, ps, *args, **kwargs): if not self.with_spans: - return fn(self, ps, *args) + return fn(self, ps, *args, **kwargs) start = ps.index - node = fn(self, ps, *args) + node = fn(self, ps, *args, **kwargs) - # Don't re-add the span if the node already has it. This may happen + # Don't re-add the span if the node already has it. This may happen # when one decorated function calls another decorated function. if node.span is not None: return node @@ -45,10 +50,8 @@ def parse(self, source): # However they should parse as standalone when they're followed by # Junk. Consequently, we only attach Comments once we know that the # Message or the Term parsed successfully. - if ( - isinstance(entry, ast.Comment) - and blank_lines == 0 and ps.current_char - ): + if isinstance(entry, ast.Comment) and len(blank_lines) == 0 \ + and ps.current_char: # Stash the comment and decide what to do with it # in the next pass. last_comment = entry @@ -161,7 +164,7 @@ def get_zero_four_style_comment(self, ps): content += ch ch = ps.take_char(lambda x: x != EOL) - if ps.is_next_line_zero_four_comment(skip=False): + if ps.is_next_line_zero_four_comment(): content += ps.current_char ps.next() ps.expect_char('/') @@ -205,7 +208,7 @@ def get_comment(self, ps): content += ch ch = ps.take_char(lambda x: x != EOL) - if ps.is_next_line_comment(skip=False, level=level): + if ps.is_next_line_comment(level=level): content += ps.current_char ps.next() else: @@ -237,44 +240,38 @@ def until_closing_bracket_or_eol(ch): @with_span def get_message(self, ps): id = self.get_identifier(ps) - ps.skip_blank_inline() - pattern = None # XXX Syntax 0.4 compat if ps.current_char == '=': ps.next() - - if ps.is_value_start(skip=True): - pattern = self.get_pattern(ps) - - if ps.is_next_line_attribute_start(skip=True): - attrs = self.get_attributes(ps) + value = self.maybe_get_pattern(ps) else: - attrs = None + value = None + + attrs = self.get_attributes(ps) - if pattern is None and attrs is None: + if value is None and len(attrs) == 0: raise ParseError('E0005', id.name) - return ast.Message(id, pattern, attrs) + return ast.Message(id, value, attrs) @with_span def get_term(self, ps): - id = self.get_term_identifier(ps) + ps.expect_char('-') + id = self.get_identifier(ps) ps.skip_blank_inline() ps.expect_char('=') - if ps.is_value_start(skip=True): - value = self.get_value(ps) - else: + # Syntax 0.8 compat: VariantLists are supported but deprecated. They + # can only be found as values of Terms. Nested VariantLists are not + # allowed. + value = self.maybe_get_variant_list(ps) or self.maybe_get_pattern(ps) + if value is None: raise ParseError('E0006', id.name) - if ps.is_next_line_attribute_start(skip=True): - attrs = self.get_attributes(ps) - else: - attrs = None - + attrs = self.get_attributes(ps) return ast.Term(id, value, attrs) @with_span @@ -286,21 +283,23 @@ def get_attribute(self, ps): ps.skip_blank_inline() ps.expect_char('=') - if ps.is_value_start(skip=True): - value = self.get_pattern(ps) - return ast.Attribute(key, value) + value = self.maybe_get_pattern(ps) + if value is None: + raise ParseError('E0012') + + return ast.Attribute(key, value) - raise ParseError('E0012') def get_attributes(self, ps): attrs = [] + ps.peek_blank() - while True: + while ps.is_attribute_start(): + ps.skip_to_peek() attr = self.get_attribute(ps) attrs.append(attr) + ps.peek_blank(); - if not ps.is_next_line_attribute_start(skip=True): - break return attrs @with_span @@ -313,12 +312,6 @@ def get_identifier(self, ps): return ast.Identifier(name) - @with_span - def get_term_identifier(self, ps): - ps.expect_char('-') - id = self.get_identifier(ps) - return ast.Identifier('-{}'.format(id.name)) - def get_variant_key(self, ps): ch = ps.current_char @@ -349,29 +342,31 @@ def get_variant(self, ps, has_default): ps.skip_blank() ps.expect_char(']') - if ps.is_value_start(skip=True): - value = self.get_value(ps) - return ast.Variant(key, value, default_index) + value = self.maybe_get_pattern(ps) + if value is None: + raise ParseError('E0012') + + return ast.Variant(key, value, default_index) - raise ParseError('E0012') def get_variants(self, ps): variants = [] has_default = False - while True: + ps.skip_blank() + while ps.is_variant_start(): variant = self.get_variant(ps, has_default) if variant.default: has_default = True variants.append(variant) - - if not ps.is_next_line_variant_start(skip=False): - break - + ps.expect_line_end() ps.skip_blank() + if len(variants) == 0: + raise ParseError('E0011') + if not has_default: raise ParseError('E0010') @@ -407,55 +402,151 @@ def get_number(self, ps): return ast.NumberLiteral(num) - @with_span - def get_value(self, ps): - if ps.current_char == '{': + def maybe_get_pattern(self, ps): + '''Parse an inline or a block Pattern, or None + + maybe_get_pattern distinguishes between patterns which start on the + same line as the indentifier (aka inline singleline patterns and inline + multiline patterns), and patterns which start on a new line (aka block + patterns). The distinction is important for the dedentation logic: the + indent of the first line of a block pattern must be taken into account + when calculating the maximum common indent. + ''' + ps.peek_blank_inline() + if ps.is_value_start(): + ps.skip_to_peek() + return self.get_pattern(ps, is_block=False) + + ps.peek_blank_block() + if ps.is_value_continuation(): + ps.skip_to_peek() + return self.get_pattern(ps, is_block=True) + + return None + + def maybe_get_variant_list(self, ps): + '''Parse a VariantList, or None + + Deprecated in Syntax 0.8. VariantLists are only allowed as values of + Terms. Values of Messages, Attributes and Variants must be Patterns. + This method is only used in get_term. + ''' + ps.peek_blank() + if ps.current_peek == '{': + start = ps.peek_offset ps.peek() ps.peek_blank_inline() - if ps.is_next_line_variant_start(skip=False): - return self.get_variant_list(ps) - ps.reset_peek() + if ps.current_peek == EOL: + ps.peek_blank() + if ps.is_variant_start(): + ps.reset_peek(start) + ps.skip_to_peek() + return self.get_variant_list(ps) - return self.get_pattern(ps) + ps.reset_peek() + return None @with_span def get_variant_list(self, ps): ps.expect_char('{') - ps.skip_blank_inline() - ps.expect_line_end() - ps.skip_blank() variants = self.get_variants(ps) - ps.expect_line_end() - ps.skip_blank() ps.expect_char('}') return ast.VariantList(variants) @with_span - def get_pattern(self, ps): + def get_pattern(self, ps, is_block): elements = [] + if is_block: + # A block pattern is a pattern which starts on a new line. Measure + # the indent of this first line for the dedentation logic. + blank_start = ps.index + first_indent = ps.skip_blank_inline() + elements.append(self.Indent(first_indent, blank_start, ps.index)) + common_indent_length = len(first_indent) + else: + common_indent_length = float('infinity') - while ps.current_char: - ch = ps.current_char - # The end condition for get_pattern's while loop is a newline - # which is not followed by a valid pattern continuation. - if ch == EOL and not ps.is_next_line_value(skip=False): + while ps.current_char: + if ps.current_char == EOL: + blank_start = ps.index + blank_lines = ps.peek_blank_block() + if ps.is_value_continuation(): + ps.skip_to_peek() + indent = ps.skip_blank_inline() + common_indent_length = min(common_indent_length, len(indent)) + elements.append(self.Indent(blank_lines + indent, blank_start, ps.index)) + continue + + # The end condition for get_pattern's while loop is a newline + # which is not followed by a valid pattern continuation. + ps.reset_peek() break - if ch == '{': + if ps.current_char == '}': + raise ParseError('E0027') + + if ps.current_char == '{': element = self.get_placeable(ps) else: element = self.get_text_element(ps) + elements.append(element) - # Trim trailing whitespace. - last_element = elements[-1] + dedented = self.dedent(elements, common_indent_length) + return ast.Pattern(dedented) + + class Indent(ast.SyntaxNode): + def __init__(self, value, start, end): + super(FluentParser.Indent, self).__init__() + self.value = value + self.add_span(start, end) + + def dedent(self, elements, common_indent): + '''Dedent a list of elements by removing the maximum common indent from + the beginning of text lines. The common indent is calculated in + get_pattern. + ''' + trimmed = [] + + for element in elements: + if isinstance(element, ast.Placeable): + trimmed.append(element) + continue + + if isinstance(element, self.Indent): + # Strip the common indent. + element.value = element.value[:len(element.value) - common_indent] + if len(element.value) == 0: + continue + + prev = trimmed[-1] if len(trimmed) > 0 else None + if isinstance(prev, ast.TextElement): + # Join adjacent TextElements by replacing them with their sum. + sum = ast.TextElement(prev.value + element.value) + if self.with_spans: + sum.add_span(prev.span.start, element.span.end) + trimmed[-1] = sum + continue + + if isinstance(element, self.Indent): + # If the indent hasn't been merged into a preceding + # TextElements, convert it into a new TextElement. + text_element = ast.TextElement(element.value) + if self.with_spans: + text_element.add_span(element.span.start, element.span.end) + element = text_element + + trimmed.append(element) + + # Trim trailing whitespace from the Pattern. + last_element = trimmed[-1] if len(trimmed) > 0 else None if isinstance(last_element, ast.TextElement): last_element.value = last_element.value.rstrip(' \t\n\r') if last_element.value == "": - elements.pop() + trimmed.pop() - return ast.Pattern(elements) + return trimmed @with_span def get_text_element(self, ps): @@ -464,62 +555,64 @@ def get_text_element(self, ps): while ps.current_char: ch = ps.current_char - if ch == '{': + if ch == '{' or ch == '}': return ast.TextElement(buf) if ch == EOL: - if not ps.is_next_line_value(skip=False): - return ast.TextElement(buf) - - ps.next() - ps.skip_blank_inline() - - buf += EOL - continue - - if ch == '\\': - ps.next() - buf += self.get_escape_sequence(ps) - continue + return ast.TextElement(buf) buf += ch ps.next() return ast.TextElement(buf) - def get_escape_sequence(self, ps, specials=('{', '\\')): + def get_escape_sequence(self, ps): next = ps.current_char - if next in specials: + if next == '\\' or next == '"': ps.next() - return '\\{}'.format(next) + return '\\{}'.format(next), next if next == 'u': - sequence = '' - ps.next() - - for _ in range(4): - ch = ps.take_hex_digit() - if not ch: - raise ParseError('E0026', sequence + ps.current_char) - sequence += ch + return self.get_unicode_escape_sequence(ps, next, 4) - return '\\u{}'.format(sequence) + if next == 'U': + return self.get_unicode_escape_sequence(ps, next, 6) raise ParseError('E0025', next) + def get_unicode_escape_sequence(self, ps, u, digits): + ps.expect_char(u) + sequence = '' + for _ in range(digits): + ch = ps.take_hex_digit() + if not ch: + raise ParseError('E0026', '\\{}{}{}'.format(u, sequence, ps.current_char)) + sequence += ch + + codepoint = int(sequence, 16) + if codepoint <= 0xD7FF or 0xE000 <= codepoint: + # It's a Unicode scalar value. + unescaped = chr(codepoint) + else: + # Escape sequences reresenting surrogate code points are + # well-formed but invalid in Fluent. Replace them with U+FFFD + # REPLACEMENT CHARACTER. + unescaped = '\uFFFD' + + return '\\{}{}'.format(u, sequence), unescaped + @with_span def get_placeable(self, ps): ps.expect_char('{') + ps.skip_blank() expression = self.get_expression(ps) ps.expect_char('}') return ast.Placeable(expression) @with_span def get_expression(self, ps): - ps.skip_blank() - - selector = self.get_selector_expression(ps) + selector = self.get_inline_expression(ps) ps.skip_blank() @@ -532,10 +625,15 @@ def get_expression(self, ps): raise ParseError('E0016') if isinstance(selector, ast.AttributeExpression) \ - and isinstance(selector.ref, ast.MessageReference): + and isinstance(selector.ref, ast.MessageReference): raise ParseError('E0018') - if isinstance(selector, ast.VariantExpression): + if isinstance(selector, ast.TermReference) \ + or isinstance(selector, ast.VariantExpression): + raise ParseError('E0017') + + if isinstance(selector, ast.CallExpression) \ + and isinstance(selector.callee, ast.TermReference): raise ParseError('E0017') ps.next() @@ -543,76 +641,89 @@ def get_expression(self, ps): ps.skip_blank_inline() ps.expect_line_end() - ps.skip_blank() variants = self.get_variants(ps) - ps.skip_blank() - - if len(variants) == 0: - raise ParseError('E0011') - - # VariantLists are only allowed in other VariantLists. - if any(isinstance(v.value, ast.VariantList) for v in variants): - raise ParseError('E0023') - return ast.SelectExpression(selector, variants) - elif ( - isinstance(selector, ast.AttributeExpression) - and isinstance(selector.ref, ast.TermReference) - ): + + if isinstance(selector, ast.AttributeExpression) \ + and isinstance(selector.ref, ast.TermReference): raise ParseError('E0019') - ps.skip_blank() + if isinstance(selector, ast.CallExpression) \ + and isinstance(selector.callee, ast.AttributeExpression): + raise ParseError('E0019') return selector @with_span - def get_selector_expression(self, ps): + def get_inline_expression(self, ps): if ps.current_char == '{': return self.get_placeable(ps) - literal = self.get_literal(ps) + expr = self.get_simple_expression(ps) - if not isinstance(literal, (ast.MessageReference, ast.TermReference)): - return literal + if isinstance(expr, (ast.NumberLiteral, ast.StringLiteral, + ast.VariableReference)): + return expr - ch = ps.current_char + if isinstance(expr, ast.MessageReference): + if ps.current_char == '.': + ps.next() + attr = self.get_identifier(ps) + return ast.AttributeExpression(expr, attr) + + if ps.current_char == '(': + # It's a Function. Ensure it's all upper-case. + if not re.match('^[A-Z][A-Z_?-]*$', expr.id.name): + raise ParseError('E0008') + func = ast.FunctionReference(expr.id) + if self.with_spans: + func.add_span(expr.span.start, expr.span.end) + return ast.CallExpression(func, *self.get_call_arguments(ps)) + + return expr + + if isinstance(expr, ast.TermReference): + if (ps.current_char == '['): + ps.next() + key = self.get_variant_key(ps) + ps.expect_char(']') + return ast.VariantExpression(expr, key) - if (ch == '.'): - ps.next() - attr = self.get_identifier(ps) - return ast.AttributeExpression(literal, attr) + if (ps.current_char == '.'): + ps.next() + attr = self.get_identifier(ps) + expr = ast.AttributeExpression(expr, attr) - if (ch == '['): - ps.next() + if (ps.current_char == '('): + return ast.CallExpression(expr, *self.get_call_arguments(ps)) - if isinstance(literal, ast.MessageReference): - raise ParseError('E0024') + return expr - key = self.get_variant_key(ps) - ps.expect_char(']') - return ast.VariantExpression(literal, key) + raise ParseError('E0028') - if (ch == '('): + @with_span + def get_simple_expression(self, ps): + if ps.is_number_start(): + return self.get_number(ps) + if ps.current_char == '"': + return self.get_string(ps) + if ps.current_char == '$': ps.next() - - if not re.match('^[A-Z][A-Z_?-]*$', literal.id.name): - raise ParseError('E0008') - - positional, named = self.get_call_args(ps) - ps.expect_char(')') - - func = ast.Function(literal.id.name) - if (self.with_spans): - func.add_span(literal.span.start, literal.span.end) - - return ast.CallExpression(func, positional, named) - - return literal + id = self.get_identifier(ps) + return ast.VariableReference(id) + if ps.current_char == '-': + ps.next() + id = self.get_identifier(ps) + return ast.TermReference(id) + if ps.is_identifier_start(): + id = self.get_identifier(ps) + return ast.MessageReference(id) + raise ParseError('E0028') @with_span - def get_call_arg(self, ps): - exp = self.get_selector_expression(ps) + def get_call_argument(self, ps): + exp = self.get_inline_expression(ps) ps.skip_blank() @@ -625,22 +736,22 @@ def get_call_arg(self, ps): ps.next() ps.skip_blank() - val = self.get_arg_val(ps) - - return ast.NamedArgument(exp.id, val) + value = self.get_literal(ps) + return ast.NamedArgument(exp.id, value) - def get_call_args(self, ps): + def get_call_arguments(self, ps): positional = [] named = [] argument_names = set() + ps.expect_char('(') ps.skip_blank() while True: if ps.current_char == ')': break - arg = self.get_call_arg(ps) + arg = self.get_call_argument(ps) if isinstance(arg, ast.NamedArgument): if arg.name.name in argument_names: raise ParseError('E0022') @@ -657,63 +768,42 @@ def get_call_args(self, ps): ps.next() ps.skip_blank() continue - else: - break - return positional, named + break - def get_arg_val(self, ps): - if ps.is_number_start(): - return self.get_number(ps) - elif ps.current_char == '"': - return self.get_string(ps) - raise ParseError('E0012') + ps.expect_char(')') + return positional, named @with_span def get_string(self, ps): - val = '' + raw = '' + value = '' ps.expect_char('"') - ch = ps.take_char(lambda x: x != '"' and x != EOL) - while ch: + while True: + ch = ps.take_char(lambda x: x != '"' and x != EOL) + if not ch: + break if ch == '\\': - val += self.get_escape_sequence(ps, ('{', '\\', '"')) + sequence, unescaped = self.get_escape_sequence(ps) + raw += sequence + value += unescaped else: - val += ch - ch = ps.take_char(lambda x: x != '"' and x != EOL) + raw += ch + value += ch if ps.current_char == EOL: raise ParseError('E0020') ps.expect_char('"') - return ast.StringLiteral(val) + return ast.StringLiteral(raw, value) @with_span def get_literal(self, ps): - ch = ps.current_char - - if ch is EOF: - raise ParseError('E0014') - - if ch == '$': - ps.next() - id = self.get_identifier(ps) - return ast.VariableReference(id) - - elif ps.is_identifier_start(): - id = self.get_identifier(ps) - return ast.MessageReference(id) - - elif ps.is_number_start(): + if ps.is_number_start(): return self.get_number(ps) - - elif ch == '-': - id = self.get_term_identifier(ps) - return ast.TermReference(id) - - elif ch == '"': + if ps.current_char == '"': return self.get_string(ps) - raise ParseError('E0014') diff --git a/fluent/syntax/serializer.py b/fluent/syntax/serializer.py index 0ae41069..4a052890 100644 --- a/fluent/syntax/serializer.py +++ b/fluent/syntax/serializer.py @@ -43,7 +43,7 @@ def serialize_entry(self, entry, state=0): if isinstance(entry, ast.Message): return serialize_message(entry) if isinstance(entry, ast.Term): - return serialize_message(entry) + return serialize_term(entry) if isinstance(entry, ast.Comment): if state & self.HAS_ENTRIES: return "\n{}\n".format(serialize_comment(entry, "#")) @@ -83,8 +83,7 @@ def serialize_message(message): if message.comment: parts.append(serialize_comment(message.comment)) - parts.append(serialize_identifier(message.id)) - parts.append(" =") + parts.append("{} =".format(message.id.name)) if message.value: parts.append(serialize_value(message.value)) @@ -94,13 +93,29 @@ def serialize_message(message): parts.append(serialize_attribute(attribute)) parts.append("\n") + return ''.join(parts) + + +def serialize_term(term): + parts = [] + + if term.comment: + parts.append(serialize_comment(term.comment)) + parts.append("-{} =".format(term.id.name)) + parts.append(serialize_value(term.value)) + + if term.attributes: + for attribute in term.attributes: + parts.append(serialize_attribute(attribute)) + + parts.append("\n") return ''.join(parts) def serialize_attribute(attribute): return "\n .{} ={}".format( - serialize_identifier(attribute.id), + attribute.id.name, indent(serialize_value(attribute.value)) ) @@ -143,16 +158,12 @@ def serialize_variant(variant): def serialize_element(element): if isinstance(element, ast.TextElement): - return serialize_text_element(element) + return element.value if isinstance(element, ast.Placeable): return serialize_placeable(element) raise Exception('Unknown element type: {}'.format(type(element))) -def serialize_text_element(text): - return text.value - - def serialize_placeable(placeable): expr = placeable.expression @@ -168,15 +179,17 @@ def serialize_placeable(placeable): def serialize_expression(expression): if isinstance(expression, ast.StringLiteral): - return serialize_string_literal(expression) + return '"{}"'.format(expression.raw) if isinstance(expression, ast.NumberLiteral): - return serialize_number_literal(expression) + return expression.value if isinstance(expression, ast.MessageReference): - return serialize_message_reference(expression) + return expression.id.name + if isinstance(expression, ast.FunctionReference): + return expression.id.name if isinstance(expression, ast.TermReference): - return serialize_message_reference(expression) + return '-{}'.format(expression.id.name) if isinstance(expression, ast.VariableReference): - return serialize_variable_reference(expression) + return '${}'.format(expression.id.name) if isinstance(expression, ast.AttributeExpression): return serialize_attribute_expression(expression) if isinstance(expression, ast.VariantExpression): @@ -190,22 +203,6 @@ def serialize_expression(expression): raise Exception('Unknown expression type: {}'.format(type(expression))) -def serialize_string_literal(expr): - return "\"{}\"".format(expr.value) - - -def serialize_number_literal(expr): - return expr.value - - -def serialize_message_reference(expr): - return serialize_identifier(expr.id) - - -def serialize_variable_reference(expr): - return "${}".format(serialize_identifier(expr.id)) - - def serialize_select_expression(expr): parts = [] selector = "{} ->".format( @@ -223,7 +220,7 @@ def serialize_select_expression(expr): def serialize_attribute_expression(expr): return "{}.{}".format( serialize_expression(expr.ref), - serialize_identifier(expr.name), + expr.name.name, ) @@ -235,53 +232,25 @@ def serialize_variant_expression(expr): def serialize_call_expression(expr): - fun = serialize_function(expr.callee) + callee = serialize_expression(expr.callee) positional = ", ".join( serialize_expression(arg) for arg in expr.positional) named = ", ".join( serialize_named_argument(arg) for arg in expr.named) if len(expr.positional) > 0 and len(expr.named) > 0: - return '{}({}, {})'.format(fun, positional, named) - return '{}({})'.format(fun, positional or named) - - -def serialize_call_argument(arg): - if isinstance(arg, ast.Expression): - return serialize_expression(arg) - if isinstance(arg, ast.NamedArgument): - return serialize_named_argument(arg) + return '{}({}, {})'.format(callee, positional, named) + return '{}({})'.format(callee, positional or named) def serialize_named_argument(arg): return "{}: {}".format( - serialize_identifier(arg.name), - serialize_argument_value(arg.value) + arg.name.name, + serialize_expression(arg.value) ) -def serialize_argument_value(argval): - if isinstance(argval, ast.StringLiteral): - return serialize_string_literal(argval) - if isinstance(argval, ast.NumberLiteral): - return serialize_number_literal(argval) - raise Exception('Unknown argument type: {}'.format(type(argval))) - - -def serialize_identifier(identifier): - return identifier.name - - -def serialize_variant_name(symbol): - return symbol.name - - def serialize_variant_key(key): if isinstance(key, ast.Identifier): - return serialize_identifier(key) - if isinstance(key, ast.NumberLiteral): - return serialize_number_literal(key) - raise Exception('Unknown variant key type: {}'.format(type(key))) - - -def serialize_function(function): - return function.name + return key.name + else: + return serialize_expression(key) diff --git a/fluent/syntax/stream.py b/fluent/syntax/stream.py index aefb24ba..161b40f6 100644 --- a/fluent/syntax/stream.py +++ b/fluent/syntax/stream.py @@ -66,47 +66,49 @@ def skip_to_peek(self): class FluentParserStream(ParserStream): last_comment_zero_four_syntax = False - def skip_blank_inline(self): - while self.current_char == ' ': - self.next() - def peek_blank_inline(self): + start = self.index + self.peek_offset while self.current_peek == ' ': self.peek() + return self.string[start:self.index + self.peek_offset] - def skip_blank_block(self): - line_count = 0 - while True: - self.peek_blank_inline() - - if self.current_peek == EOL: - self.skip_to_peek() - self.next() - line_count += 1 - else: - self.reset_peek() - return line_count + def skip_blank_inline(self): + blank = self.peek_blank_inline() + self.skip_to_peek() + return blank def peek_blank_block(self): + blank = "" while True: line_start = self.peek_offset - self.peek_blank_inline() if self.current_peek == EOL: + blank += EOL self.peek() - else: - self.reset_peek(line_start) - break + continue - def skip_blank(self): - while self.current_char in (" ", EOL): - self.next() + if self.current_peek is EOF: + # Treat the blank line at EOF as a blank block. + return blank + + # Any other char; reset to column 1 on this line. + self.reset_peek(line_start) + return blank + + def skip_blank_block(self): + blank = self.peek_blank_block() + self.skip_to_peek() + return blank def peek_blank(self): while self.current_peek in (" ", EOL): self.peek() + def skip_blank(self): + self.peek_blank() + self.skip_to_peek() + def expect_char(self, ch): if self.current_char == ch: self.next() @@ -163,24 +165,28 @@ def is_char_pattern_continuation(self, ch): return ch not in SPECIAL_LINE_START_CHARS - def is_value_start(self, skip): - if skip is False: - raise NotImplementedError() + def is_value_start(self): + # Inline Patterns may start with any char. + return self.current_peek is not EOF and self.current_peek != EOL + def is_value_continuation(self): + column1 = self.peek_offset self.peek_blank_inline() - ch = self.current_peek - # Inline Patterns may start with any char. - if ch is not EOF and ch != EOL: - self.skip_to_peek() + if self.current_peek == '{': + self.reset_peek(column1) return True - return self.is_next_line_value(skip) + if self.peek_offset - column1 == 0: + return False - def is_next_line_zero_four_comment(self, skip): - if skip is True: - raise NotImplementedError() + if self.is_char_pattern_continuation(self.current_peek): + self.reset_peek(column1) + return True + return False + + def is_next_line_zero_four_comment(self): if self.current_peek != EOL: return False @@ -192,10 +198,7 @@ def is_next_line_zero_four_comment(self, skip): # 0 - comment # 1 - group comment # 2 - resource comment - def is_next_line_comment(self, skip, level=-1): - if skip is True: - raise NotImplementedError() - + def is_next_line_comment(self, level=-1): if self.current_peek != EOL: return False @@ -217,63 +220,19 @@ def is_next_line_comment(self, skip, level=-1): self.reset_peek() return False - def is_next_line_variant_start(self, skip): - if skip is True: - raise NotImplementedError() - - if self.current_peek != EOL: - return False - - self.peek_blank() - + def is_variant_start(self): + current_peek_offset = self.peek_offset if self.current_peek == '*': self.peek() - if self.current_peek == '[' and self.peek() != '[': - self.reset_peek() - return True - - self.reset_peek() - return False - - def is_next_line_attribute_start(self, skip): - if skip is False: - raise NotImplementedError() - - self.peek_blank() - - if self.current_peek == '.': - self.skip_to_peek() + self.reset_peek(current_peek_offset) return True - self.reset_peek() + self.reset_peek(current_peek_offset) return False - def is_next_line_value(self, skip): - if self.current_peek != EOL: - return False - - self.peek_blank_block() - - ptr = self.peek_offset - - self.peek_blank_inline() - - if self.current_peek != "{": - if (self.peek_offset - ptr == 0): - self.reset_peek() - return False - - if not self.is_char_pattern_continuation(self.current_peek): - self.reset_peek() - return False - - if skip: - self.skip_to_peek() - else: - self.reset_peek() - - return True + def is_attribute_start(self): + return self.current_peek == '.' def skip_to_next_entry_start(self, junk_start): last_newline = self.string.rfind(EOL, 0, self.index) diff --git a/tests/syntax/fixtures_behavior/call_expression_with_bad_id.ftl b/tests/syntax/fixtures_behavior/call_expression_with_bad_id.ftl index e8214696..9bc2bbbc 100644 --- a/tests/syntax/fixtures_behavior/call_expression_with_bad_id.ftl +++ b/tests/syntax/fixtures_behavior/call_expression_with_bad_id.ftl @@ -1,3 +1,3 @@ key = { no-caps-name() } -# ~ERROR E0008, pos 21 +# ~ERROR E0008, pos 20 diff --git a/tests/syntax/fixtures_behavior/call_expression_with_wrong_value_type.ftl b/tests/syntax/fixtures_behavior/call_expression_with_wrong_value_type.ftl index 4a8f9604..b0be5cde 100644 --- a/tests/syntax/fixtures_behavior/call_expression_with_wrong_value_type.ftl +++ b/tests/syntax/fixtures_behavior/call_expression_with_wrong_value_type.ftl @@ -1,2 +1,2 @@ key = { BUILTIN(key: foo) } -# ~ERROR E0012, pos 21 +# ~ERROR E0014, pos 21 diff --git a/tests/syntax/fixtures_behavior/escape_sequences.ftl b/tests/syntax/fixtures_behavior/escape_sequences.ftl index d0bbb85a..19ad1329 100644 --- a/tests/syntax/fixtures_behavior/escape_sequences.ftl +++ b/tests/syntax/fixtures_behavior/escape_sequences.ftl @@ -1,10 +1,16 @@ -# ~ERROR E0025, pos 8, args "A" -key1 = \A +## Backslash is a regular character in text elements. +key01 = \A +key02 = \u0041 +key03 = \\u0041 +key04 = \u000z +key05 = \{Value} -# ~ERROR E0026, pos 23, args "000z" -key2 = \u000z +key06 = {"Escaped \" quote"} +key07 = {"Escaped \\ backslash"} +key08 = {"Escaped \u0041 A"} -key3 = \{Escaped} -key4 = {"Escaped \" quote"} -key5 = \u0041 -key6 = \\u0041 +# ~ERROR E0025, pos 232, args "A" +key09 = {"\A"} + +# ~ERROR E0026, pos 252, args "\u000z" +key10 = {"\u000z"} diff --git a/tests/syntax/fixtures_behavior/placeable_in_placeable.ftl b/tests/syntax/fixtures_behavior/placeable_in_placeable.ftl index 7ece456b..827c93e7 100644 --- a/tests/syntax/fixtures_behavior/placeable_in_placeable.ftl +++ b/tests/syntax/fixtures_behavior/placeable_in_placeable.ftl @@ -7,8 +7,9 @@ key2 = { { foo } } # { foo } # } -key4 = { { foo } # ~ERROR E0003, pos 96, args "}" +key4 = { { foo } +# ~ERROR E0027, pos 111 key5 = { foo } } diff --git a/tests/syntax/fixtures_behavior/select_expression_without_variants.ftl b/tests/syntax/fixtures_behavior/select_expression_without_variants.ftl index 7e43355d..8eea7b42 100644 --- a/tests/syntax/fixtures_behavior/select_expression_without_variants.ftl +++ b/tests/syntax/fixtures_behavior/select_expression_without_variants.ftl @@ -3,4 +3,4 @@ key = { $foo -> } key = { $foo -> } -# ~ERROR E0003, pos 39, args "[" +# ~ERROR E0011, pos 39 diff --git a/tests/syntax/fixtures_behavior/selector_expression_ends_abruptly.ftl b/tests/syntax/fixtures_behavior/selector_expression_ends_abruptly.ftl index a11a1814..8cb33985 100644 --- a/tests/syntax/fixtures_behavior/selector_expression_ends_abruptly.ftl +++ b/tests/syntax/fixtures_behavior/selector_expression_ends_abruptly.ftl @@ -1,2 +1,2 @@ key = { $foo -> -# ~ERROR E0003, pos 16, args "[" +# ~ERROR E0011, pos 16 diff --git a/tests/syntax/fixtures_behavior/term.ftl b/tests/syntax/fixtures_behavior/term.ftl index 272e3039..303225fd 100644 --- a/tests/syntax/fixtures_behavior/term.ftl +++ b/tests/syntax/fixtures_behavior/term.ftl @@ -11,22 +11,21 @@ key2 = key3 = Test { -brand-short-name[accusative] } +key4 = { -brand() } + +# ~ERROR E0004, pos 306, args "0-9" err1 = { $foo -> [one] Foo *[-other] Foo 2 } -# ~ERROR E0004, pos 285, args "0-9" +# ~ERROR E0004, pos 336, args "a-zA-Z" err2 = { $-foo } -# ~ERROR E0004, pos 315, args "a-zA-Z" - -err4 = { -brand() } -# ~ERROR E0008, pos 339 +# ~ERROR E0006, pos 351, args "err5" -err5 = -# ~ERROR E0006, pos 351, args "-err5" +# ~ERROR E0006, pos 360, args "err6" -err6 = .attr = Attribute -# ~ERROR E0006, pos 360, args "-err6" diff --git a/tests/syntax/fixtures_behavior/unclosed_empty_placeable_error.ftl b/tests/syntax/fixtures_behavior/unclosed_empty_placeable_error.ftl index 5f812a94..e29ced2b 100644 --- a/tests/syntax/fixtures_behavior/unclosed_empty_placeable_error.ftl +++ b/tests/syntax/fixtures_behavior/unclosed_empty_placeable_error.ftl @@ -1,5 +1,5 @@ # ~ERROR E0003, pos 8, args "}" foo = { bar = Bar -# ~ERROR E0014, pos 26 +# ~ERROR E0028, pos 26 baz = { diff --git a/tests/syntax/fixtures_behavior/variant_expression_as_placeable.ftl b/tests/syntax/fixtures_behavior/variant_expression_as_placeable.ftl index 1866ef64..6b2796ff 100644 --- a/tests/syntax/fixtures_behavior/variant_expression_as_placeable.ftl +++ b/tests/syntax/fixtures_behavior/variant_expression_as_placeable.ftl @@ -1,3 +1,3 @@ -# ~ERROR E0024, pos 18 +# ~ERROR E0003, pos 17, args "}" key01 = { message[variant] } key02 = { -term[variant] } diff --git a/tests/syntax/fixtures_behavior/variant_expression_as_selector.ftl b/tests/syntax/fixtures_behavior/variant_expression_as_selector.ftl index 88d6b5a2..946c1c83 100644 --- a/tests/syntax/fixtures_behavior/variant_expression_as_selector.ftl +++ b/tests/syntax/fixtures_behavior/variant_expression_as_selector.ftl @@ -3,7 +3,7 @@ err1 = *[1] One [2] Two } -# ~ERROR E0024, pos 17 +# ~ERROR E0003, pos 16, args "}" err2 = { -foo[bar] -> diff --git a/tests/syntax/fixtures_behavior/variant_lists.ftl b/tests/syntax/fixtures_behavior/variant_lists.ftl index 10f5cab3..681751de 100644 --- a/tests/syntax/fixtures_behavior/variant_lists.ftl +++ b/tests/syntax/fixtures_behavior/variant_lists.ftl @@ -1,10 +1,10 @@ -# ~ERROR E0014, pos 25 +# ~ERROR E0028, pos 25 message1 = { *[one] One } -# ~ERROR E0023, pos 123 +# ~ERROR E0028, pos 97 message2 = { $sel -> *[one] { @@ -17,6 +17,7 @@ message2 = *[one] One } +# ~ERROR E0028, pos 211 -term2 = { *[one] { @@ -24,7 +25,7 @@ message2 = } } -# ~ERROR E0023, pos 318 +# ~ERROR E0028, pos 292 -term3 = { $sel -> *[one] { diff --git a/tests/syntax/fixtures_structure/blank_lines.ftl b/tests/syntax/fixtures_structure/blank_lines.ftl new file mode 100644 index 00000000..0d207399 --- /dev/null +++ b/tests/syntax/fixtures_structure/blank_lines.ftl @@ -0,0 +1,27 @@ +### NOTE: Disable final newline insertion and trimming when editing this file. + +key01 = Value 01 + +key02 = Value 02 + + +key03 = + + Value 03 + + Continued + +# There are four spaces on the line between "Value 04" and "Continued". +key04 = + + Value 04 + + Continued + +# There are four spaces on the line following "Value 05". +key05 = + Value 05 + +# There are four spaces on the line following "Value 06". +key06 = Value 06 + \ No newline at end of file diff --git a/tests/syntax/fixtures_structure/blank_lines.json b/tests/syntax/fixtures_structure/blank_lines.json new file mode 100644 index 00000000..998e7da6 --- /dev/null +++ b/tests/syntax/fixtures_structure/blank_lines.json @@ -0,0 +1,271 @@ +{ + "type": "Resource", + "body": [ + { + "type": "ResourceComment", + "content": "NOTE: Disable final newline insertion and trimming when editing this file.", + "span": { + "type": "Span", + "start": 0, + "end": 78 + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key01", + "span": { + "type": "Span", + "start": 80, + "end": 85 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value 01", + "span": { + "type": "Span", + "start": 88, + "end": 96 + } + } + ], + "span": { + "type": "Span", + "start": 88, + "end": 96 + } + }, + "attributes": [], + "comment": null, + "span": { + "type": "Span", + "start": 80, + "end": 96 + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key02", + "span": { + "type": "Span", + "start": 98, + "end": 103 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value 02", + "span": { + "type": "Span", + "start": 106, + "end": 114 + } + } + ], + "span": { + "type": "Span", + "start": 106, + "end": 114 + } + }, + "attributes": [], + "comment": null, + "span": { + "type": "Span", + "start": 98, + "end": 114 + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key03", + "span": { + "type": "Span", + "start": 117, + "end": 122 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value 03\n\nContinued", + "span": { + "type": "Span", + "start": 130, + "end": 153 + } + } + ], + "span": { + "type": "Span", + "start": 126, + "end": 153 + } + }, + "attributes": [], + "comment": null, + "span": { + "type": "Span", + "start": 117, + "end": 153 + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key04", + "span": { + "type": "Span", + "start": 227, + "end": 232 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value 04\n\nContinued", + "span": { + "type": "Span", + "start": 240, + "end": 267 + } + } + ], + "span": { + "type": "Span", + "start": 236, + "end": 267 + } + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "There are four spaces on the line between \"Value 04\" and \"Continued\".", + "span": { + "type": "Span", + "start": 155, + "end": 226 + } + }, + "span": { + "type": "Span", + "start": 155, + "end": 267 + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key05", + "span": { + "type": "Span", + "start": 327, + "end": 332 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value 05", + "span": { + "type": "Span", + "start": 339, + "end": 347 + } + } + ], + "span": { + "type": "Span", + "start": 335, + "end": 347 + } + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "There are four spaces on the line following \"Value 05\".", + "span": { + "type": "Span", + "start": 269, + "end": 326 + } + }, + "span": { + "type": "Span", + "start": 269, + "end": 347 + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key06", + "span": { + "type": "Span", + "start": 411, + "end": 416 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value 06", + "span": { + "type": "Span", + "start": 419, + "end": 427 + } + } + ], + "span": { + "type": "Span", + "start": 419, + "end": 427 + } + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "There are four spaces on the line following \"Value 06\".", + "span": { + "type": "Span", + "start": 353, + "end": 410 + } + }, + "span": { + "type": "Span", + "start": 353, + "end": 427 + } + } + ], + "span": { + "type": "Span", + "start": 0, + "end": 432 + } +} diff --git a/tests/syntax/fixtures_structure/crlf.json b/tests/syntax/fixtures_structure/crlf.json index b72afa33..9461bfaa 100644 --- a/tests/syntax/fixtures_structure/crlf.json +++ b/tests/syntax/fixtures_structure/crlf.json @@ -65,7 +65,7 @@ ], "span": { "type": "Span", - "start": 35, + "start": 31, "end": 58 } }, diff --git a/tests/syntax/fixtures_structure/escape_sequences.ftl b/tests/syntax/fixtures_structure/escape_sequences.ftl index e91de86b..5242a4bc 100644 --- a/tests/syntax/fixtures_structure/escape_sequences.ftl +++ b/tests/syntax/fixtures_structure/escape_sequences.ftl @@ -1,9 +1,22 @@ -backslash = Value with \\ (an escaped backslash) -closing-brace = Value with \{ (an opening brace) -unicode-escape = \u0041 -escaped-unicode = \\u0041 +## Literal text +text-backslash-one = Value with \ a backslash +text-backslash-two = Value with \\ two backslashes +text-backslash-brace = Value with \{placeable} +text-backslash-u = \u0041 +text-backslash-backslash-u = \\u0041 -## String Expressions +## String literals quote-in-string = {"\""} backslash-in-string = {"\\"} +# ERROR Mismatched quote mismatched-quote = {"\\""} +# ERROR Unknown escape +unknown-escape = {"\x"} + +## Unicode escapes +string-unicode-sequence = {"\u0041"} +string-escaped-unicode = {"\\u0041"} + +## Literal braces +brace-open = An opening {"{"} brace. +brace-close = A closing {"}"} brace. diff --git a/tests/syntax/fixtures_structure/escape_sequences.json b/tests/syntax/fixtures_structure/escape_sequences.json index f49683f6..dccf18cb 100644 --- a/tests/syntax/fixtures_structure/escape_sequences.json +++ b/tests/syntax/fixtures_structure/escape_sequences.json @@ -1,15 +1,24 @@ { "type": "Resource", "body": [ + { + "type": "GroupComment", + "content": "Literal text", + "span": { + "type": "Span", + "start": 0, + "end": 15 + } + }, { "type": "Message", "id": { "type": "Identifier", - "name": "backslash", + "name": "text-backslash-one", "span": { "type": "Span", - "start": 0, - "end": 9 + "start": 16, + "end": 34 } }, "value": { @@ -17,37 +26,37 @@ "elements": [ { "type": "TextElement", - "value": "Value with \\\\ (an escaped backslash)", + "value": "Value with \\ a backslash", "span": { "type": "Span", - "start": 12, - "end": 48 + "start": 37, + "end": 61 } } ], "span": { "type": "Span", - "start": 12, - "end": 48 + "start": 37, + "end": 61 } }, "attributes": [], "comment": null, "span": { "type": "Span", - "start": 0, - "end": 48 + "start": 16, + "end": 61 } }, { "type": "Message", "id": { "type": "Identifier", - "name": "closing-brace", + "name": "text-backslash-two", "span": { "type": "Span", - "start": 49, - "end": 62 + "start": 62, + "end": 80 } }, "value": { @@ -55,37 +64,100 @@ "elements": [ { "type": "TextElement", - "value": "Value with \\{ (an opening brace)", + "value": "Value with \\\\ two backslashes", "span": { "type": "Span", - "start": 65, - "end": 97 + "start": 83, + "end": 112 } } ], "span": { "type": "Span", - "start": 65, - "end": 97 + "start": 83, + "end": 112 } }, "attributes": [], "comment": null, "span": { "type": "Span", - "start": 49, - "end": 97 + "start": 62, + "end": 112 } }, { "type": "Message", "id": { "type": "Identifier", - "name": "unicode-escape", + "name": "text-backslash-brace", "span": { "type": "Span", - "start": 98, - "end": 112 + "start": 113, + "end": 133 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value with \\", + "span": { + "type": "Span", + "start": 136, + "end": 148 + } + }, + { + "type": "Placeable", + "expression": { + "type": "MessageReference", + "id": { + "type": "Identifier", + "name": "placeable", + "span": { + "type": "Span", + "start": 149, + "end": 158 + } + }, + "span": { + "type": "Span", + "start": 149, + "end": 158 + } + }, + "span": { + "type": "Span", + "start": 148, + "end": 159 + } + } + ], + "span": { + "type": "Span", + "start": 136, + "end": 159 + } + }, + "attributes": [], + "comment": null, + "span": { + "type": "Span", + "start": 113, + "end": 159 + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "text-backslash-u", + "span": { + "type": "Span", + "start": 160, + "end": 176 } }, "value": { @@ -96,34 +168,34 @@ "value": "\\u0041", "span": { "type": "Span", - "start": 115, - "end": 121 + "start": 179, + "end": 185 } } ], "span": { "type": "Span", - "start": 115, - "end": 121 + "start": 179, + "end": 185 } }, "attributes": [], "comment": null, "span": { "type": "Span", - "start": 98, - "end": 121 + "start": 160, + "end": 185 } }, { "type": "Message", "id": { "type": "Identifier", - "name": "escaped-unicode", + "name": "text-backslash-backslash-u", "span": { "type": "Span", - "start": 122, - "end": 137 + "start": 186, + "end": 212 } }, "value": { @@ -134,32 +206,32 @@ "value": "\\\\u0041", "span": { "type": "Span", - "start": 140, - "end": 147 + "start": 215, + "end": 222 } } ], "span": { "type": "Span", - "start": 140, - "end": 147 + "start": 215, + "end": 222 } }, "attributes": [], "comment": null, "span": { "type": "Span", - "start": 122, - "end": 147 + "start": 186, + "end": 222 } }, { "type": "GroupComment", - "content": "String Expressions", + "content": "String literals", "span": { "type": "Span", - "start": 149, - "end": 170 + "start": 224, + "end": 242 } }, { @@ -169,8 +241,8 @@ "name": "quote-in-string", "span": { "type": "Span", - "start": 171, - "end": 186 + "start": 243, + "end": 258 } }, "value": { @@ -180,32 +252,33 @@ "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\\"", + "raw": "\\\"", + "value": "\"", "span": { "type": "Span", - "start": 190, - "end": 194 + "start": 262, + "end": 266 } }, "span": { "type": "Span", - "start": 189, - "end": 195 + "start": 261, + "end": 267 } } ], "span": { "type": "Span", - "start": 189, - "end": 195 + "start": 261, + "end": 267 } }, "attributes": [], "comment": null, "span": { "type": "Span", - "start": 171, - "end": 195 + "start": 243, + "end": 267 } }, { @@ -215,8 +288,8 @@ "name": "backslash-in-string", "span": { "type": "Span", - "start": 196, - "end": 215 + "start": 268, + "end": 287 } }, "value": { @@ -226,32 +299,42 @@ "type": "Placeable", "expression": { "type": "StringLiteral", - "value": "\\\\", + "raw": "\\\\", + "value": "\\", "span": { "type": "Span", - "start": 219, - "end": 223 + "start": 291, + "end": 295 } }, "span": { "type": "Span", - "start": 218, - "end": 224 + "start": 290, + "end": 296 } } ], "span": { "type": "Span", - "start": 218, - "end": 224 + "start": 290, + "end": 296 } }, "attributes": [], "comment": null, "span": { "type": "Span", - "start": 196, - "end": 224 + "start": 268, + "end": 296 + } + }, + { + "type": "Comment", + "content": "ERROR Mismatched quote", + "span": { + "type": "Span", + "start": 297, + "end": 321 } }, { @@ -266,22 +349,297 @@ "message": "Expected token: \"}\"", "span": { "type": "Span", - "start": 249, - "end": 249 + "start": 346, + "end": 346 } } ], "content": "mismatched-quote = {\"\\\\\"\"}\n", "span": { "type": "Span", - "start": 225, - "end": 252 + "start": 322, + "end": 349 + } + }, + { + "type": "Comment", + "content": "ERROR Unknown escape", + "span": { + "type": "Span", + "start": 349, + "end": 371 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0025", + "args": [ + "x" + ], + "message": "Unknown escape sequence: \\x.", + "span": { + "type": "Span", + "start": 392, + "end": 392 + } + } + ], + "content": "unknown-escape = {\"\\x\"}\n\n", + "span": { + "type": "Span", + "start": 372, + "end": 397 + } + }, + { + "type": "GroupComment", + "content": "Unicode escapes", + "span": { + "type": "Span", + "start": 397, + "end": 415 + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "string-unicode-sequence", + "span": { + "type": "Span", + "start": 416, + "end": 439 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\u0041", + "value": "A", + "span": { + "type": "Span", + "start": 443, + "end": 451 + } + }, + "span": { + "type": "Span", + "start": 442, + "end": 452 + } + } + ], + "span": { + "type": "Span", + "start": 442, + "end": 452 + } + }, + "attributes": [], + "comment": null, + "span": { + "type": "Span", + "start": 416, + "end": 452 + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "string-escaped-unicode", + "span": { + "type": "Span", + "start": 453, + "end": 475 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\\\u0041", + "value": "\\u0041", + "span": { + "type": "Span", + "start": 479, + "end": 488 + } + }, + "span": { + "type": "Span", + "start": 478, + "end": 489 + } + } + ], + "span": { + "type": "Span", + "start": 478, + "end": 489 + } + }, + "attributes": [], + "comment": null, + "span": { + "type": "Span", + "start": 453, + "end": 489 + } + }, + { + "type": "GroupComment", + "content": "Literal braces", + "span": { + "type": "Span", + "start": 491, + "end": 508 + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "brace-open", + "span": { + "type": "Span", + "start": 509, + "end": 519 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "An opening ", + "span": { + "type": "Span", + "start": 522, + "end": 533 + } + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "{", + "value": "{", + "span": { + "type": "Span", + "start": 534, + "end": 537 + } + }, + "span": { + "type": "Span", + "start": 533, + "end": 538 + } + }, + { + "type": "TextElement", + "value": " brace.", + "span": { + "type": "Span", + "start": 538, + "end": 545 + } + } + ], + "span": { + "type": "Span", + "start": 522, + "end": 545 + } + }, + "attributes": [], + "comment": null, + "span": { + "type": "Span", + "start": 509, + "end": 545 + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "brace-close", + "span": { + "type": "Span", + "start": 546, + "end": 557 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "A closing ", + "span": { + "type": "Span", + "start": 560, + "end": 570 + } + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "}", + "value": "}", + "span": { + "type": "Span", + "start": 571, + "end": 574 + } + }, + "span": { + "type": "Span", + "start": 570, + "end": 575 + } + }, + { + "type": "TextElement", + "value": " brace.", + "span": { + "type": "Span", + "start": 575, + "end": 582 + } + } + ], + "span": { + "type": "Span", + "start": 560, + "end": 582 + } + }, + "attributes": [], + "comment": null, + "span": { + "type": "Span", + "start": 546, + "end": 582 } } ], "span": { "type": "Span", "start": 0, - "end": 252 + "end": 583 } } diff --git a/tests/syntax/fixtures_structure/expressions_call_args.json b/tests/syntax/fixtures_structure/expressions_call_args.json index a4ea0b03..c7be7f94 100644 --- a/tests/syntax/fixtures_structure/expressions_call_args.json +++ b/tests/syntax/fixtures_structure/expressions_call_args.json @@ -20,8 +20,16 @@ "expression": { "type": "CallExpression", "callee": { - "type": "Function", - "name": "FOO", + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FOO", + "span": { + "type": "Span", + "start": 8, + "end": 11 + } + }, "span": { "type": "Span", "start": 8, diff --git a/tests/syntax/fixtures_structure/junk.ftl b/tests/syntax/fixtures_structure/junk.ftl new file mode 100644 index 00000000..0ce05ea9 --- /dev/null +++ b/tests/syntax/fixtures_structure/junk.ftl @@ -0,0 +1,23 @@ +err01 = {1xx} +err02 = {1xx} + +err03 = {1xx} +1xx + +err04 = {1xx} + +1xx + +err05 = { + +1xx + +err06 = {1xx + + .attr = Value + +err07 = { + +key08 = Value + +err09 = { diff --git a/tests/syntax/fixtures_structure/junk.json b/tests/syntax/fixtures_structure/junk.json new file mode 100644 index 00000000..df1f862d --- /dev/null +++ b/tests/syntax/fixtures_structure/junk.json @@ -0,0 +1,238 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0003", + "args": [ + "}" + ], + "message": "Expected token: \"}\"", + "span": { + "type": "Span", + "start": 10, + "end": 10 + } + } + ], + "content": "err01 = {1xx}\n", + "span": { + "type": "Span", + "start": 0, + "end": 14 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0003", + "args": [ + "}" + ], + "message": "Expected token: \"}\"", + "span": { + "type": "Span", + "start": 24, + "end": 24 + } + } + ], + "content": "err02 = {1xx}\n\n", + "span": { + "type": "Span", + "start": 14, + "end": 29 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0003", + "args": [ + "}" + ], + "message": "Expected token: \"}\"", + "span": { + "type": "Span", + "start": 39, + "end": 39 + } + } + ], + "content": "err03 = {1xx}\n1xx\n\n", + "span": { + "type": "Span", + "start": 29, + "end": 48 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0003", + "args": [ + "}" + ], + "message": "Expected token: \"}\"", + "span": { + "type": "Span", + "start": 58, + "end": 58 + } + } + ], + "content": "err04 = {1xx}\n\n1xx\n\n", + "span": { + "type": "Span", + "start": 48, + "end": 68 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0003", + "args": [ + "}" + ], + "message": "Expected token: \"}\"", + "span": { + "type": "Span", + "start": 80, + "end": 80 + } + } + ], + "content": "err05 = {\n\n1xx\n\n", + "span": { + "type": "Span", + "start": 68, + "end": 84 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0003", + "args": [ + "}" + ], + "message": "Expected token: \"}\"", + "span": { + "type": "Span", + "start": 94, + "end": 94 + } + } + ], + "content": "err06 = {1xx\n\n .attr = Value\n\n", + "span": { + "type": "Span", + "start": 84, + "end": 117 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0003", + "args": [ + "}" + ], + "message": "Expected token: \"}\"", + "span": { + "type": "Span", + "start": 128, + "end": 128 + } + } + ], + "content": "err07 = {\n\n", + "span": { + "type": "Span", + "start": 117, + "end": 128 + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key08", + "span": { + "type": "Span", + "start": 128, + "end": 133 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value", + "span": { + "type": "Span", + "start": 136, + "end": 141 + } + } + ], + "span": { + "type": "Span", + "start": 136, + "end": 141 + } + }, + "attributes": [], + "comment": null, + "span": { + "type": "Span", + "start": 128, + "end": 141 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0028", + "args": [], + "message": "Expected an inline expression", + "span": { + "type": "Span", + "start": 153, + "end": 153 + } + } + ], + "content": "err09 = {\n", + "span": { + "type": "Span", + "start": 143, + "end": 153 + } + } + ], + "span": { + "type": "Span", + "start": 0, + "end": 153 + } +} diff --git a/tests/syntax/fixtures_structure/leading_dots.json b/tests/syntax/fixtures_structure/leading_dots.json index 837427b4..f95994c1 100644 --- a/tests/syntax/fixtures_structure/leading_dots.json +++ b/tests/syntax/fixtures_structure/leading_dots.json @@ -95,6 +95,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": ".", "span": { "type": "Span", @@ -150,6 +151,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": ".", "span": { "type": "Span", @@ -175,7 +177,7 @@ ], "span": { "type": "Span", - "start": 61, + "start": 57, "end": 71 } }, @@ -214,6 +216,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": ".", "span": { "type": "Span", @@ -278,6 +281,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": ".", "span": { "type": "Span", @@ -489,6 +493,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": ".", "span": { "type": "Span", @@ -514,7 +519,7 @@ ], "span": { "type": "Span", - "start": 428, + "start": 424, "end": 484 } }, @@ -565,7 +570,7 @@ ], "span": { "type": "Span", - "start": 515, + "start": 511, "end": 516 } }, @@ -671,6 +676,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": ".", "span": { "type": "Span", @@ -696,7 +702,7 @@ ], "span": { "type": "Span", - "start": 589, + "start": 580, "end": 599 } }, @@ -797,6 +803,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": ".", "value": ".", "span": { "type": "Span", @@ -822,7 +829,7 @@ ], "span": { "type": "Span", - "start": 669, + "start": 657, "end": 679 } }, @@ -836,7 +843,7 @@ ], "span": { "type": "Span", - "start": 614, + "start": 615, "end": 684 } }, @@ -849,7 +856,7 @@ ], "span": { "type": "Span", - "start": 613, + "start": 609, "end": 685 } }, diff --git a/tests/syntax/fixtures_structure/multiline_pattern.json b/tests/syntax/fixtures_structure/multiline_pattern.json index 0df2d450..03c51cf3 100644 --- a/tests/syntax/fixtures_structure/multiline_pattern.json +++ b/tests/syntax/fixtures_structure/multiline_pattern.json @@ -70,7 +70,7 @@ } ], "span": { - "start": 47, + "start": 43, "end": 72, "type": "Span" } @@ -116,7 +116,7 @@ } ], "span": { - "start": 171, + "start": 167, "end": 176, "type": "Span" } @@ -186,7 +186,7 @@ } ], "span": { - "start": 311, + "start": 307, "end": 316, "type": "Span" } diff --git a/tests/syntax/fixtures_structure/placeable_at_eol.json b/tests/syntax/fixtures_structure/placeable_at_eol.json index f79d7454..7018303e 100644 --- a/tests/syntax/fixtures_structure/placeable_at_eol.json +++ b/tests/syntax/fixtures_structure/placeable_at_eol.json @@ -61,7 +61,7 @@ ], "span": { "type": "Span", - "start": 11, + "start": 7, "end": 131 } }, @@ -124,7 +124,7 @@ ], "span": { "type": "Span", - "start": 144, + "start": 140, "end": 184 } }, diff --git a/tests/syntax/fixtures_structure/select_expressions.ftl b/tests/syntax/fixtures_structure/select_expressions.ftl new file mode 100644 index 00000000..5a96bf90 --- /dev/null +++ b/tests/syntax/fixtures_structure/select_expressions.ftl @@ -0,0 +1,9 @@ +# ERROR No blanks are allowed between * and [. +err01 = { $sel -> + * [key] Value +} + +# ERROR Missing default variant. +err02 = { $sel -> + [key] Value +} diff --git a/tests/syntax/fixtures_structure/select_expressions.json b/tests/syntax/fixtures_structure/select_expressions.json new file mode 100644 index 00000000..8cee7ae3 --- /dev/null +++ b/tests/syntax/fixtures_structure/select_expressions.json @@ -0,0 +1,72 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Comment", + "content": "ERROR No blanks are allowed between * and [.", + "span": { + "type": "Span", + "start": 0, + "end": 46 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0011", + "args": [], + "message": "Expected at least one variant after \"->\"", + "span": { + "type": "Span", + "start": 69, + "end": 69 + } + } + ], + "content": "err01 = { $sel ->\n * [key] Value\n}\n\n", + "span": { + "type": "Span", + "start": 47, + "end": 87 + } + }, + { + "type": "Comment", + "content": "ERROR Missing default variant.", + "span": { + "type": "Span", + "start": 87, + "end": 119 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0010", + "args": [], + "message": "Expected one of the variants to be marked as default (*)", + "span": { + "type": "Span", + "start": 154, + "end": 154 + } + } + ], + "content": "err02 = { $sel ->\n [key] Value\n}\n", + "span": { + "type": "Span", + "start": 120, + "end": 156 + } + } + ], + "span": { + "type": "Span", + "start": 0, + "end": 156 + } +} diff --git a/tests/syntax/fixtures_structure/sparse-messages.json b/tests/syntax/fixtures_structure/sparse-messages.json index 90f8fdb3..32780f68 100644 --- a/tests/syntax/fixtures_structure/sparse-messages.json +++ b/tests/syntax/fixtures_structure/sparse-messages.json @@ -27,7 +27,7 @@ ], "span": { "type": "Span", - "start": 12, + "start": 8, "end": 17 } }, @@ -122,7 +122,7 @@ ], "span": { "type": "Span", - "start": 63, + "start": 59, "end": 104 } }, @@ -322,7 +322,7 @@ ], "span": { "type": "Span", - "start": 154, + "start": 155, "end": 208 } }, @@ -351,10 +351,10 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-key7", + "name": "key7", "span": { "type": "Span", - "start": 211, + "start": 212, "end": 216 } }, @@ -417,10 +417,10 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-key8", + "name": "key8", "span": { "type": "Span", - "start": 254, + "start": 255, "end": 259 } }, diff --git a/tests/syntax/fixtures_structure/term.json b/tests/syntax/fixtures_structure/term.json index f9cf94d4..c36ec450 100644 --- a/tests/syntax/fixtures_structure/term.json +++ b/tests/syntax/fixtures_structure/term.json @@ -5,10 +5,10 @@ "type": "Term", "id": { "type": "Identifier", - "name": "-brand-name", + "name": "brand-name", "span": { "type": "Span", - "start": 0, + "start": 1, "end": 11 } }, @@ -172,10 +172,10 @@ "type": "TermReference", "id": { "type": "Identifier", - "name": "-brand-name", + "name": "brand-name", "span": { "type": "Span", - "start": 145, + "start": 146, "end": 156 } }, @@ -218,7 +218,7 @@ ], "span": { "type": "Span", - "start": 131, + "start": 127, "end": 171 } }, @@ -254,10 +254,10 @@ "type": "TermReference", "id": { "type": "Identifier", - "name": "-brand-name", + "name": "brand-name", "span": { "type": "Span", - "start": 199, + "start": 200, "end": 210 } }, @@ -303,10 +303,10 @@ "type": "TermReference", "id": { "type": "Identifier", - "name": "-brand-name", + "name": "brand-name", "span": { "type": "Span", - "start": 243, + "start": 244, "end": 254 } }, @@ -365,10 +365,10 @@ "type": "TermReference", "id": { "type": "Identifier", - "name": "-brand-name", + "name": "brand-name", "span": { "type": "Span", - "start": 311, + "start": 312, "end": 322 } }, @@ -436,10 +436,10 @@ "type": "TermReference", "id": { "type": "Identifier", - "name": "-brand-name", + "name": "brand-name", "span": { "type": "Span", - "start": 385, + "start": 386, "end": 396 } }, @@ -481,7 +481,7 @@ ], "span": { "type": "Span", - "start": 198, + "start": 199, "end": 436 } }, @@ -494,7 +494,7 @@ ], "span": { "type": "Span", - "start": 197, + "start": 193, "end": 437 } }, diff --git a/tests/syntax/fixtures_structure/term_with_empty_pattern.json b/tests/syntax/fixtures_structure/term_with_empty_pattern.json index 66b7e06b..13e018ec 100644 --- a/tests/syntax/fixtures_structure/term_with_empty_pattern.json +++ b/tests/syntax/fixtures_structure/term_with_empty_pattern.json @@ -8,7 +8,7 @@ "type": "Annotation", "code": "E0006", "args": [ - "-foo" + "foo" ], "message": "Expected term \"-foo\" to have a value", "span": { @@ -32,7 +32,7 @@ "type": "Annotation", "code": "E0006", "args": [ - "-bar" + "bar" ], "message": "Expected term \"-bar\" to have a value", "span": { diff --git a/tests/syntax/fixtures_structure/variant_keys.ftl b/tests/syntax/fixtures_structure/variant_keys.ftl new file mode 100644 index 00000000..fc0f241c --- /dev/null +++ b/tests/syntax/fixtures_structure/variant_keys.ftl @@ -0,0 +1,61 @@ +key01 = { $sel -> + *[ + key + ] Value +} + +key02 = { $sel -> + *[ + key + ] + + Value +} + +err01 = { $sel -> + *["key"] Value +} + +err02 = { $sel -> + *[-key] Value +} + +err03 = { $sel -> + *[-key.attr] Value +} + +err04 = { $sel -> + *[-key()] Value +} + +err05 = { $sel -> + *[-key.attr()] Value +} + +err06 = { $sel -> + *[key.attr] Value +} + +err07 = { $sel -> + *[$key] Value +} + +err08 = { $sel -> + *[FUNC()] Value +} + +err09 = { $sel -> + *[{key}] Value +} + +err10 = { $sel -> + *[{"key"}] Value +} + +err11 = { $sel -> + *[{3.14}] Value +} + +err12 = { $sel -> + *[{$key}] Value +} diff --git a/tests/syntax/fixtures_structure/variant_keys.json b/tests/syntax/fixtures_structure/variant_keys.json new file mode 100644 index 00000000..0115ee51 --- /dev/null +++ b/tests/syntax/fixtures_structure/variant_keys.json @@ -0,0 +1,500 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key01", + "span": { + "type": "Span", + "start": 0, + "end": 5 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "sel", + "span": { + "type": "Span", + "start": 11, + "end": 14 + } + }, + "span": { + "type": "Span", + "start": 10, + "end": 14 + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key", + "span": { + "type": "Span", + "start": 33, + "end": 36 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value", + "span": { + "type": "Span", + "start": 43, + "end": 48 + } + } + ], + "span": { + "type": "Span", + "start": 43, + "end": 48 + } + }, + "default": true, + "span": { + "type": "Span", + "start": 22, + "end": 48 + } + } + ], + "span": { + "type": "Span", + "start": 10, + "end": 49 + } + }, + "span": { + "type": "Span", + "start": 8, + "end": 50 + } + } + ], + "span": { + "type": "Span", + "start": 8, + "end": 50 + } + }, + "attributes": [], + "comment": null, + "span": { + "type": "Span", + "start": 0, + "end": 50 + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key02", + "span": { + "type": "Span", + "start": 52, + "end": 57 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "sel", + "span": { + "type": "Span", + "start": 63, + "end": 66 + } + }, + "span": { + "type": "Span", + "start": 62, + "end": 66 + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key", + "span": { + "type": "Span", + "start": 85, + "end": 88 + } + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value", + "span": { + "type": "Span", + "start": 104, + "end": 109 + } + } + ], + "span": { + "type": "Span", + "start": 100, + "end": 109 + } + }, + "default": true, + "span": { + "type": "Span", + "start": 74, + "end": 109 + } + } + ], + "span": { + "type": "Span", + "start": 62, + "end": 110 + } + }, + "span": { + "type": "Span", + "start": 60, + "end": 111 + } + } + ], + "span": { + "type": "Span", + "start": 60, + "end": 111 + } + }, + "attributes": [], + "comment": null, + "span": { + "type": "Span", + "start": 52, + "end": 111 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0004", + "args": [ + "a-zA-Z" + ], + "message": "Expected a character from range: \"a-zA-Z\"", + "span": { + "type": "Span", + "start": 137, + "end": 137 + } + } + ], + "content": "err01 = { $sel ->\n *[\"key\"] Value\n}\n\n", + "span": { + "type": "Span", + "start": 113, + "end": 153 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0004", + "args": [ + "0-9" + ], + "message": "Expected a character from range: \"0-9\"", + "span": { + "type": "Span", + "start": 178, + "end": 178 + } + } + ], + "content": "err02 = { $sel ->\n *[-key] Value\n}\n\n", + "span": { + "type": "Span", + "start": 153, + "end": 192 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0004", + "args": [ + "0-9" + ], + "message": "Expected a character from range: \"0-9\"", + "span": { + "type": "Span", + "start": 217, + "end": 217 + } + } + ], + "content": "err03 = { $sel ->\n *[-key.attr] Value\n}\n\n", + "span": { + "type": "Span", + "start": 192, + "end": 236 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0004", + "args": [ + "0-9" + ], + "message": "Expected a character from range: \"0-9\"", + "span": { + "type": "Span", + "start": 261, + "end": 261 + } + } + ], + "content": "err04 = { $sel ->\n *[-key()] Value\n}\n\n", + "span": { + "type": "Span", + "start": 236, + "end": 277 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0004", + "args": [ + "0-9" + ], + "message": "Expected a character from range: \"0-9\"", + "span": { + "type": "Span", + "start": 302, + "end": 302 + } + } + ], + "content": "err05 = { $sel ->\n *[-key.attr()] Value\n}\n\n", + "span": { + "type": "Span", + "start": 277, + "end": 323 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0003", + "args": [ + "]" + ], + "message": "Expected token: \"]\"", + "span": { + "type": "Span", + "start": 350, + "end": 350 + } + } + ], + "content": "err06 = { $sel ->\n *[key.attr] Value\n}\n\n", + "span": { + "type": "Span", + "start": 323, + "end": 366 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0004", + "args": [ + "a-zA-Z" + ], + "message": "Expected a character from range: \"a-zA-Z\"", + "span": { + "type": "Span", + "start": 390, + "end": 390 + } + } + ], + "content": "err07 = { $sel ->\n *[$key] Value\n}\n\n", + "span": { + "type": "Span", + "start": 366, + "end": 405 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0003", + "args": [ + "]" + ], + "message": "Expected token: \"]\"", + "span": { + "type": "Span", + "start": 433, + "end": 433 + } + } + ], + "content": "err08 = { $sel ->\n *[FUNC()] Value\n}\n\n", + "span": { + "type": "Span", + "start": 405, + "end": 446 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0004", + "args": [ + "a-zA-Z" + ], + "message": "Expected a character from range: \"a-zA-Z\"", + "span": { + "type": "Span", + "start": 470, + "end": 470 + } + } + ], + "content": "err09 = { $sel ->\n *[{key}] Value\n}\n\n", + "span": { + "type": "Span", + "start": 446, + "end": 486 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0004", + "args": [ + "a-zA-Z" + ], + "message": "Expected a character from range: \"a-zA-Z\"", + "span": { + "type": "Span", + "start": 510, + "end": 510 + } + } + ], + "content": "err10 = { $sel ->\n *[{\"key\"}] Value\n}\n\n", + "span": { + "type": "Span", + "start": 486, + "end": 528 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0004", + "args": [ + "a-zA-Z" + ], + "message": "Expected a character from range: \"a-zA-Z\"", + "span": { + "type": "Span", + "start": 552, + "end": 552 + } + } + ], + "content": "err11 = { $sel ->\n *[{3.14}] Value\n}\n\n", + "span": { + "type": "Span", + "start": 528, + "end": 569 + } + }, + { + "type": "Junk", + "annotations": [ + { + "type": "Annotation", + "code": "E0004", + "args": [ + "a-zA-Z" + ], + "message": "Expected a character from range: \"a-zA-Z\"", + "span": { + "type": "Span", + "start": 593, + "end": 593 + } + } + ], + "content": "err12 = { $sel ->\n *[{$key}] Value\n}\n", + "span": { + "type": "Span", + "start": 569, + "end": 609 + } + } + ], + "span": { + "type": "Span", + "start": 0, + "end": 609 + } +} diff --git a/tests/syntax/fixtures_structure/variant_with_empty_pattern.json b/tests/syntax/fixtures_structure/variant_with_empty_pattern.json index 19fe137d..c1d77e6d 100644 --- a/tests/syntax/fixtures_structure/variant_with_empty_pattern.json +++ b/tests/syntax/fixtures_structure/variant_with_empty_pattern.json @@ -47,6 +47,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "", "value": "", "span": { "type": "Span", @@ -77,7 +78,7 @@ ], "span": { "type": "Span", - "start": 12, + "start": 13, "end": 41 } }, @@ -90,7 +91,7 @@ ], "span": { "type": "Span", - "start": 11, + "start": 7, "end": 42 } }, diff --git a/tests/syntax/fixtures_structure/whitespace_leading.json b/tests/syntax/fixtures_structure/whitespace_leading.json index 168eee92..1ee9e575 100644 --- a/tests/syntax/fixtures_structure/whitespace_leading.json +++ b/tests/syntax/fixtures_structure/whitespace_leading.json @@ -111,6 +111,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": "", "value": "", "span": { "type": "Span", @@ -166,6 +167,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": " ", "value": " ", "span": { "type": "Span", diff --git a/tests/syntax/fixtures_structure/whitespace_trailing.json b/tests/syntax/fixtures_structure/whitespace_trailing.json index 69e33deb..0a58feab 100644 --- a/tests/syntax/fixtures_structure/whitespace_trailing.json +++ b/tests/syntax/fixtures_structure/whitespace_trailing.json @@ -192,6 +192,7 @@ "type": "Placeable", "expression": { "type": "StringLiteral", + "raw": " ", "value": " ", "span": { "type": "Span", diff --git a/tests/syntax/test_serializer.py b/tests/syntax/test_serializer.py index 08570b14..e9d3831d 100644 --- a/tests/syntax/test_serializer.py +++ b/tests/syntax/test_serializer.py @@ -423,15 +423,21 @@ def test_call_expression_with_positional_and_named_arguments(self): """ self.assertEqual(self.pretty_ftl(input), dedent_ftl(input)) + def test_macro_call(self): + input = """\ + foo = { -term() } + """ + self.assertEqual(self.pretty_ftl(input), dedent_ftl(input)) + def test_nested_placeables(self): input = """\ foo = {{ FOO() }} """ self.assertEqual(self.pretty_ftl(input), dedent_ftl(input)) - def test_escaped_special_in_text_element(self): + def test_backslash_in_text(self): input = """\ - foo = \\{Escaped} + foo = \\{ placeable } """ self.assertEqual(self.pretty_ftl(input), dedent_ftl(input)) @@ -443,7 +449,7 @@ def test_escaped_special_in_string_literal(self): def test_escaped_unicode_sequence(self): input = """\ - foo = \\u0065 + foo = { "\\u0065" } """ self.assertEqual(self.pretty_ftl(input), dedent_ftl(input)) From c1241586aba0305dcb62185dff0e892b3d6715aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Thu, 6 Dec 2018 11:22:13 -0500 Subject: [PATCH 2/5] Use ImportError rather than ModuleNotFoundError --- fluent/syntax/parser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fluent/syntax/parser.py b/fluent/syntax/parser.py index 57ebf8d0..a904aaf7 100644 --- a/fluent/syntax/parser.py +++ b/fluent/syntax/parser.py @@ -6,10 +6,11 @@ try: from __builtin__ import unichr as chr -except ModuleNotFoundError: +except ImportError: pass + def with_span(fn): def decorated(self, ps, *args, **kwargs): if not self.with_spans: From 2e912015562cf7633c7268715693ff15bafc760b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Thu, 6 Dec 2018 11:14:59 -0500 Subject: [PATCH 3/5] Test reference fixtures --- fluent/syntax/ast.py | 13 +- tests/syntax/fixtures_reference/any_char.ftl | 8 + tests/syntax/fixtures_reference/any_char.json | 68 + tests/syntax/fixtures_reference/astral.ftl | 20 + tests/syntax/fixtures_reference/astral.json | 178 +++ .../fixtures_reference/call_expressions.ftl | 104 ++ .../fixtures_reference/call_expressions.json | 1139 +++++++++++++++++ .../fixtures_reference/callee_expressions.ftl | 46 + .../callee_expressions.json | 270 ++++ tests/syntax/fixtures_reference/comments.ftl | 15 + tests/syntax/fixtures_reference/comments.json | 63 + tests/syntax/fixtures_reference/cr.ftl | 1 + tests/syntax/fixtures_reference/cr.json | 9 + tests/syntax/fixtures_reference/crlf.ftl | 14 + tests/syntax/fixtures_reference/crlf.json | 76 ++ .../syntax/fixtures_reference/eof_comment.ftl | 3 + .../fixtures_reference/eof_comment.json | 13 + tests/syntax/fixtures_reference/eof_empty.ftl | 0 .../syntax/fixtures_reference/eof_empty.json | 4 + tests/syntax/fixtures_reference/eof_id.ftl | 3 + tests/syntax/fixtures_reference/eof_id.json | 14 + .../fixtures_reference/eof_id_equals.ftl | 3 + .../fixtures_reference/eof_id_equals.json | 14 + tests/syntax/fixtures_reference/eof_junk.ftl | 3 + tests/syntax/fixtures_reference/eof_junk.json | 14 + tests/syntax/fixtures_reference/eof_value.ftl | 3 + .../syntax/fixtures_reference/eof_value.json | 27 + .../fixtures_reference/escaped_characters.ftl | 34 + .../escaped_characters.json | 399 ++++++ tests/syntax/fixtures_reference/junk.ftl | 21 + tests/syntax/fixtures_reference/junk.json | 68 + .../fixtures_reference/leading_dots.ftl | 76 ++ .../fixtures_reference/leading_dots.json | 487 +++++++ .../literal_expressions.ftl | 3 + .../literal_expressions.json | 69 + .../fixtures_reference/member_expressions.ftl | 28 + .../member_expressions.json | 173 +++ tests/syntax/fixtures_reference/messages.ftl | 29 + tests/syntax/fixtures_reference/messages.json | 267 ++++ .../fixtures_reference/mixed_entries.ftl | 24 + .../fixtures_reference/mixed_entries.json | 136 ++ .../fixtures_reference/multiline_values.ftl | 60 + .../fixtures_reference/multiline_values.json | 329 +++++ .../syntax/fixtures_reference/placeables.ftl | 15 + .../syntax/fixtures_reference/placeables.json | 113 ++ .../reference_expressions.ftl | 28 + .../reference_expressions.json | 185 +++ .../fixtures_reference/select_expressions.ftl | 53 + .../select_expressions.json | 294 +++++ .../fixtures_reference/select_indent.ftl | 95 ++ .../fixtures_reference/select_indent.json | 688 ++++++++++ .../fixtures_reference/sparse_entries.ftl | 39 + .../fixtures_reference/sparse_entries.json | 163 +++ tests/syntax/fixtures_reference/tab.ftl | 14 + tests/syntax/fixtures_reference/tab.json | 70 + .../fixtures_reference/term_parameters.ftl | 8 + .../fixtures_reference/term_parameters.json | 203 +++ tests/syntax/fixtures_reference/terms.ftl | 23 + tests/syntax/fixtures_reference/terms.json | 107 ++ tests/syntax/fixtures_reference/variables.ftl | 17 + .../syntax/fixtures_reference/variables.json | 132 ++ .../fixtures_reference/variant_keys.ftl | 37 + .../fixtures_reference/variant_keys.json | 156 +++ .../fixtures_reference/variant_lists.ftl | 55 + .../fixtures_reference/variant_lists.json | 180 +++ .../fixtures_reference/variants_indent.ftl | 19 + .../fixtures_reference/variants_indent.json | 146 +++ .../whitespace_in_value.ftl | 10 + .../whitespace_in_value.json | 26 + tests/syntax/test_reference.py | 64 + 70 files changed, 7262 insertions(+), 6 deletions(-) create mode 100644 tests/syntax/fixtures_reference/any_char.ftl create mode 100644 tests/syntax/fixtures_reference/any_char.json create mode 100644 tests/syntax/fixtures_reference/astral.ftl create mode 100644 tests/syntax/fixtures_reference/astral.json create mode 100644 tests/syntax/fixtures_reference/call_expressions.ftl create mode 100644 tests/syntax/fixtures_reference/call_expressions.json create mode 100644 tests/syntax/fixtures_reference/callee_expressions.ftl create mode 100644 tests/syntax/fixtures_reference/callee_expressions.json create mode 100644 tests/syntax/fixtures_reference/comments.ftl create mode 100644 tests/syntax/fixtures_reference/comments.json create mode 100644 tests/syntax/fixtures_reference/cr.ftl create mode 100644 tests/syntax/fixtures_reference/cr.json create mode 100644 tests/syntax/fixtures_reference/crlf.ftl create mode 100644 tests/syntax/fixtures_reference/crlf.json create mode 100644 tests/syntax/fixtures_reference/eof_comment.ftl create mode 100644 tests/syntax/fixtures_reference/eof_comment.json create mode 100644 tests/syntax/fixtures_reference/eof_empty.ftl create mode 100644 tests/syntax/fixtures_reference/eof_empty.json create mode 100644 tests/syntax/fixtures_reference/eof_id.ftl create mode 100644 tests/syntax/fixtures_reference/eof_id.json create mode 100644 tests/syntax/fixtures_reference/eof_id_equals.ftl create mode 100644 tests/syntax/fixtures_reference/eof_id_equals.json create mode 100644 tests/syntax/fixtures_reference/eof_junk.ftl create mode 100644 tests/syntax/fixtures_reference/eof_junk.json create mode 100644 tests/syntax/fixtures_reference/eof_value.ftl create mode 100644 tests/syntax/fixtures_reference/eof_value.json create mode 100644 tests/syntax/fixtures_reference/escaped_characters.ftl create mode 100644 tests/syntax/fixtures_reference/escaped_characters.json create mode 100644 tests/syntax/fixtures_reference/junk.ftl create mode 100644 tests/syntax/fixtures_reference/junk.json create mode 100644 tests/syntax/fixtures_reference/leading_dots.ftl create mode 100644 tests/syntax/fixtures_reference/leading_dots.json create mode 100644 tests/syntax/fixtures_reference/literal_expressions.ftl create mode 100644 tests/syntax/fixtures_reference/literal_expressions.json create mode 100644 tests/syntax/fixtures_reference/member_expressions.ftl create mode 100644 tests/syntax/fixtures_reference/member_expressions.json create mode 100644 tests/syntax/fixtures_reference/messages.ftl create mode 100644 tests/syntax/fixtures_reference/messages.json create mode 100644 tests/syntax/fixtures_reference/mixed_entries.ftl create mode 100644 tests/syntax/fixtures_reference/mixed_entries.json create mode 100644 tests/syntax/fixtures_reference/multiline_values.ftl create mode 100644 tests/syntax/fixtures_reference/multiline_values.json create mode 100644 tests/syntax/fixtures_reference/placeables.ftl create mode 100644 tests/syntax/fixtures_reference/placeables.json create mode 100644 tests/syntax/fixtures_reference/reference_expressions.ftl create mode 100644 tests/syntax/fixtures_reference/reference_expressions.json create mode 100644 tests/syntax/fixtures_reference/select_expressions.ftl create mode 100644 tests/syntax/fixtures_reference/select_expressions.json create mode 100644 tests/syntax/fixtures_reference/select_indent.ftl create mode 100644 tests/syntax/fixtures_reference/select_indent.json create mode 100644 tests/syntax/fixtures_reference/sparse_entries.ftl create mode 100644 tests/syntax/fixtures_reference/sparse_entries.json create mode 100644 tests/syntax/fixtures_reference/tab.ftl create mode 100644 tests/syntax/fixtures_reference/tab.json create mode 100644 tests/syntax/fixtures_reference/term_parameters.ftl create mode 100644 tests/syntax/fixtures_reference/term_parameters.json create mode 100644 tests/syntax/fixtures_reference/terms.ftl create mode 100644 tests/syntax/fixtures_reference/terms.json create mode 100644 tests/syntax/fixtures_reference/variables.ftl create mode 100644 tests/syntax/fixtures_reference/variables.json create mode 100644 tests/syntax/fixtures_reference/variant_keys.ftl create mode 100644 tests/syntax/fixtures_reference/variant_keys.json create mode 100644 tests/syntax/fixtures_reference/variant_lists.ftl create mode 100644 tests/syntax/fixtures_reference/variant_lists.json create mode 100644 tests/syntax/fixtures_reference/variants_indent.ftl create mode 100644 tests/syntax/fixtures_reference/variants_indent.json create mode 100644 tests/syntax/fixtures_reference/whitespace_in_value.ftl create mode 100644 tests/syntax/fixtures_reference/whitespace_in_value.json create mode 100644 tests/syntax/test_reference.py diff --git a/fluent/syntax/ast.py b/fluent/syntax/ast.py index 4feaa841..b7ee24bb 100644 --- a/fluent/syntax/ast.py +++ b/fluent/syntax/ast.py @@ -3,13 +3,13 @@ import json -def to_json(value): +def to_json(value, **kwargs): if isinstance(value, BaseNode): - return value.to_json() + return value.to_json(**kwargs) if isinstance(value, list): - return list(map(to_json, value)) + return list(to_json(item, **kwargs) for item in value) if isinstance(value, tuple): - return list(map(to_json, value)) + return list(to_json(item, **kwargs) for item in value) else: return value @@ -119,10 +119,11 @@ def equals(self, other, ignored_fields=['span']): return True - def to_json(self): + def to_json(self, with_spans=True): obj = { - name: to_json(value) + name: to_json(value, with_spans=with_spans) for name, value in vars(self).items() + if with_spans or name != 'span' } obj.update( {'type': self.__class__.__name__} diff --git a/tests/syntax/fixtures_reference/any_char.ftl b/tests/syntax/fixtures_reference/any_char.ftl new file mode 100644 index 00000000..6966a0da --- /dev/null +++ b/tests/syntax/fixtures_reference/any_char.ftl @@ -0,0 +1,8 @@ +# ↓ BEL, U+0007 +control0 = abcdef + +# ↓ DEL, U+007F +delete = abcdef + +# ↓ BPM, U+0082 +control1 = abc‚def diff --git a/tests/syntax/fixtures_reference/any_char.json b/tests/syntax/fixtures_reference/any_char.json new file mode 100644 index 00000000..07e7dc4b --- /dev/null +++ b/tests/syntax/fixtures_reference/any_char.json @@ -0,0 +1,68 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "control0" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "abc\u0007def" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": " ↓ BEL, U+0007" + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "delete" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "abcdef" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": " ↓ DEL, U+007F" + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "control1" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "abc‚def" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": " ↓ BPM, U+0082" + } + } + ] +} diff --git a/tests/syntax/fixtures_reference/astral.ftl b/tests/syntax/fixtures_reference/astral.ftl new file mode 100644 index 00000000..b77e32e3 --- /dev/null +++ b/tests/syntax/fixtures_reference/astral.ftl @@ -0,0 +1,20 @@ +face-with-tears-of-joy = 😂 +tetragram-for-centre = 𝌆 + +surrogates-in-text = \uD83D\uDE02 +surrogates-in-string = {"\uD83D\uDE02"} +surrogates-in-adjacent-strings = {"\uD83D"}{"\uDE02"} + +emoji-in-text = A face 😂 with tears of joy. +emoji-in-string = {"A face 😂 with tears of joy."} + +# ERROR Invalid identifier +err-😂 = Value + +# ERROR Invalid expression +err-invalid-expression = { 😂 } + +# ERROR Invalid variant key +err-invalid-variant-key = { $sel -> + *[😂] Value +} diff --git a/tests/syntax/fixtures_reference/astral.json b/tests/syntax/fixtures_reference/astral.json new file mode 100644 index 00000000..b69743ca --- /dev/null +++ b/tests/syntax/fixtures_reference/astral.json @@ -0,0 +1,178 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "face-with-tears-of-joy" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "😂" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "tetragram-for-centre" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "𝌆" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "surrogates-in-text" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "\\uD83D\\uDE02" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "surrogates-in-string" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\uD83D\\uDE02", + "value": "��" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "surrogates-in-adjacent-strings" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\uD83D", + "value": "�" + } + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\uDE02", + "value": "�" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "emoji-in-text" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "A face 😂 with tears of joy." + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "emoji-in-string" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "A face 😂 with tears of joy.", + "value": "A face 😂 with tears of joy." + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR Invalid identifier" + }, + { + "type": "Junk", + "annotations": [], + "content": "err-😂 = Value\n\n" + }, + { + "type": "Comment", + "content": "ERROR Invalid expression" + }, + { + "type": "Junk", + "annotations": [], + "content": "err-invalid-expression = { 😂 }\n\n" + }, + { + "type": "Comment", + "content": "ERROR Invalid variant key" + }, + { + "type": "Junk", + "annotations": [], + "content": "err-invalid-variant-key = { $sel ->\n *[😂] Value\n}\n" + } + ] +} diff --git a/tests/syntax/fixtures_reference/call_expressions.ftl b/tests/syntax/fixtures_reference/call_expressions.ftl new file mode 100644 index 00000000..19e76b7e --- /dev/null +++ b/tests/syntax/fixtures_reference/call_expressions.ftl @@ -0,0 +1,104 @@ +## Arguments + +positional-args = {FUN(1, "a", msg)} +named-args = {FUN(x: 1, y: "Y")} +dense-named-args = {FUN(x:1, y:"Y")} +mixed-args = {FUN(1, "a", msg, x: 1, y: "Y")} + +# ERROR Positional arg must not follow keyword args +shuffled-args = {FUN(1, x: 1, "a", y: "Y", msg)} + +# ERROR Named arguments must be unique +duplicate-named-args = {FUN(x: 1, x: "X")} + + +## Whitespace around arguments + +sparse-inline-call = {FUN( "a" , msg, x: 1 )} +empty-inline-call = {FUN( )} +multiline-call = {FUN( + "a", + msg, + x: 1 + )} +sparse-multiline-call = {FUN( + + "a" , + msg + , x: 1 + )} +empty-multiline-call = {FUN( + + )} + + +unindented-arg-number = {FUN( +1)} + +unindented-arg-string = {FUN( +"a")} + +unindented-arg-msg-ref = {FUN( +msg)} + +unindented-arg-term-ref = {FUN( +-msg)} + +unindented-arg-var-ref = {FUN( +$var)} + +unindented-arg-call = {FUN( +OTHER())} + +unindented-named-arg = {FUN( +x:1)} + +unindented-closing-paren = {FUN( + x +)} + + + +## Optional trailing comma + +one-argument = {FUN(1,)} +many-arguments = {FUN(1, 2, 3,)} +inline-sparse-args = {FUN( 1, 2, 3, )} +mulitline-args = {FUN( + 1, + 2, + )} +mulitline-sparse-args = {FUN( + + 1 + , + 2 + , + )} + + +## Syntax errors for trailing comma + +one-argument = {FUN(1,,)} +missing-arg = {FUN(,)} +missing-sparse-arg = {FUN( , )} + + +## Whitespace in named arguments + +sparse-named-arg = {FUN( + x : 1, + y : 2, + z + : + 3 + )} + + +unindented-colon = {FUN( + x +:1)} + +unindented-value = {FUN( + x: +1)} diff --git a/tests/syntax/fixtures_reference/call_expressions.json b/tests/syntax/fixtures_reference/call_expressions.json new file mode 100644 index 00000000..4dd73483 --- /dev/null +++ b/tests/syntax/fixtures_reference/call_expressions.json @@ -0,0 +1,1139 @@ +{ + "type": "Resource", + "body": [ + { + "type": "GroupComment", + "content": "Arguments" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "positional-args" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "NumberLiteral", + "value": "1" + }, + { + "type": "StringLiteral", + "raw": "a", + "value": "a" + }, + { + "type": "MessageReference", + "id": { + "type": "Identifier", + "name": "msg" + } + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "named-args" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [], + "named": [ + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "x" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + }, + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "y" + }, + "value": { + "type": "StringLiteral", + "raw": "Y", + "value": "Y" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "dense-named-args" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [], + "named": [ + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "x" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + }, + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "y" + }, + "value": { + "type": "StringLiteral", + "raw": "Y", + "value": "Y" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "mixed-args" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "NumberLiteral", + "value": "1" + }, + { + "type": "StringLiteral", + "raw": "a", + "value": "a" + }, + { + "type": "MessageReference", + "id": { + "type": "Identifier", + "name": "msg" + } + } + ], + "named": [ + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "x" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + }, + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "y" + }, + "value": { + "type": "StringLiteral", + "raw": "Y", + "value": "Y" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR Positional arg must not follow keyword args" + }, + { + "type": "Junk", + "annotations": [], + "content": "shuffled-args = {FUN(1, x: 1, \"a\", y: \"Y\", msg)}\n\n" + }, + { + "type": "Comment", + "content": "ERROR Named arguments must be unique" + }, + { + "type": "Junk", + "annotations": [], + "content": "duplicate-named-args = {FUN(x: 1, x: \"X\")}\n\n\n" + }, + { + "type": "GroupComment", + "content": "Whitespace around arguments" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "sparse-inline-call" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "StringLiteral", + "raw": "a", + "value": "a" + }, + { + "type": "MessageReference", + "id": { + "type": "Identifier", + "name": "msg" + } + } + ], + "named": [ + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "x" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "empty-inline-call" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "multiline-call" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "StringLiteral", + "raw": "a", + "value": "a" + }, + { + "type": "MessageReference", + "id": { + "type": "Identifier", + "name": "msg" + } + } + ], + "named": [ + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "x" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "sparse-multiline-call" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "StringLiteral", + "raw": "a", + "value": "a" + }, + { + "type": "MessageReference", + "id": { + "type": "Identifier", + "name": "msg" + } + } + ], + "named": [ + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "x" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "empty-multiline-call" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "unindented-arg-number" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "NumberLiteral", + "value": "1" + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "unindented-arg-string" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "StringLiteral", + "raw": "a", + "value": "a" + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "unindented-arg-msg-ref" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "MessageReference", + "id": { + "type": "Identifier", + "name": "msg" + } + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "unindented-arg-term-ref" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "msg" + } + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "unindented-arg-var-ref" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "var" + } + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "unindented-arg-call" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "OTHER" + } + }, + "positional": [], + "named": [] + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "unindented-named-arg" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [], + "named": [ + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "x" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "unindented-closing-paren" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "MessageReference", + "id": { + "type": "Identifier", + "name": "x" + } + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "GroupComment", + "content": "Optional trailing comma" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "one-argument" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "NumberLiteral", + "value": "1" + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "many-arguments" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "NumberLiteral", + "value": "1" + }, + { + "type": "NumberLiteral", + "value": "2" + }, + { + "type": "NumberLiteral", + "value": "3" + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "inline-sparse-args" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "NumberLiteral", + "value": "1" + }, + { + "type": "NumberLiteral", + "value": "2" + }, + { + "type": "NumberLiteral", + "value": "3" + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "mulitline-args" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "NumberLiteral", + "value": "1" + }, + { + "type": "NumberLiteral", + "value": "2" + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "mulitline-sparse-args" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [ + { + "type": "NumberLiteral", + "value": "1" + }, + { + "type": "NumberLiteral", + "value": "2" + } + ], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "GroupComment", + "content": "Syntax errors for trailing comma" + }, + { + "type": "Junk", + "annotations": [], + "content": "one-argument = {FUN(1,,)}\n" + }, + { + "type": "Junk", + "annotations": [], + "content": "missing-arg = {FUN(,)}\n" + }, + { + "type": "Junk", + "annotations": [], + "content": "missing-sparse-arg = {FUN( , )}\n\n\n" + }, + { + "type": "GroupComment", + "content": "Whitespace in named arguments" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "sparse-named-arg" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [], + "named": [ + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "x" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + }, + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "y" + }, + "value": { + "type": "NumberLiteral", + "value": "2" + } + }, + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "z" + }, + "value": { + "type": "NumberLiteral", + "value": "3" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "unindented-colon" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [], + "named": [ + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "x" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "unindented-value" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + }, + "positional": [], + "named": [ + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "x" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + } + ] +} diff --git a/tests/syntax/fixtures_reference/callee_expressions.ftl b/tests/syntax/fixtures_reference/callee_expressions.ftl new file mode 100644 index 00000000..637a2e4d --- /dev/null +++ b/tests/syntax/fixtures_reference/callee_expressions.ftl @@ -0,0 +1,46 @@ +## Callees in placeables. + +function-callee-placeable = {FUNCTION()} +term-callee-placeable = {-term()} + +# ERROR Messages cannot be parameterized. +message-callee-placeable = {message()} +# ERROR Equivalent to a MessageReference callee. +mixed-case-callee-placeable = {Function()} +# ERROR Message attributes cannot be parameterized. +message-attr-callee-placeable = {message.attr()} +# ERROR Term attributes may not be used in Placeables. +term-attr-callee-placeable = {-term.attr()} +# ERROR Variables cannot be parameterized. +variable-callee-placeable = {$variable()} + + +## Callees in selectors. + +function-callee-selector = {FUNCTION() -> + *[key] Value +} +term-attr-callee-selector = {-term.attr() -> + *[key] Value +} + +# ERROR Messages cannot be parameterized. +message-callee-selector = {message() -> + *[key] Value +} +# ERROR Equivalent to a MessageReference callee. +mixed-case-callee-selector = {Function() -> + *[key] Value +} +# ERROR Message attributes cannot be parameterized. +message-attr-callee-selector = {message.attr() -> + *[key] Value +} +# ERROR Term values may not be used as selectors. +term-callee-selector = {-term() -> + *[key] Value +} +# ERROR Variables cannot be parameterized. +variable-callee-selector = {$variable() -> + *[key] Value +} diff --git a/tests/syntax/fixtures_reference/callee_expressions.json b/tests/syntax/fixtures_reference/callee_expressions.json new file mode 100644 index 00000000..50cdaeb4 --- /dev/null +++ b/tests/syntax/fixtures_reference/callee_expressions.json @@ -0,0 +1,270 @@ +{ + "type": "Resource", + "body": [ + { + "type": "GroupComment", + "content": "Callees in placeables." + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "function-callee-placeable" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUNCTION" + } + }, + "positional": [], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "term-callee-placeable" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + }, + "positional": [], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR Messages cannot be parameterized." + }, + { + "type": "Junk", + "annotations": [], + "content": "message-callee-placeable = {message()}\n" + }, + { + "type": "Comment", + "content": "ERROR Equivalent to a MessageReference callee." + }, + { + "type": "Junk", + "annotations": [], + "content": "mixed-case-callee-placeable = {Function()}\n" + }, + { + "type": "Comment", + "content": "ERROR Message attributes cannot be parameterized." + }, + { + "type": "Junk", + "annotations": [], + "content": "message-attr-callee-placeable = {message.attr()}\n" + }, + { + "type": "Comment", + "content": "ERROR Term attributes may not be used in Placeables." + }, + { + "type": "Junk", + "annotations": [], + "content": "term-attr-callee-placeable = {-term.attr()}\n" + }, + { + "type": "Comment", + "content": "ERROR Variables cannot be parameterized." + }, + { + "type": "Junk", + "annotations": [], + "content": "variable-callee-placeable = {$variable()}\n\n\n" + }, + { + "type": "GroupComment", + "content": "Callees in selectors." + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "function-callee-selector" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "FUNCTION" + } + }, + "positional": [], + "named": [] + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "term-attr-callee-selector" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "CallExpression", + "callee": { + "type": "AttributeExpression", + "ref": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + }, + "name": { + "type": "Identifier", + "name": "attr" + } + }, + "positional": [], + "named": [] + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR Messages cannot be parameterized." + }, + { + "type": "Junk", + "annotations": [], + "content": "message-callee-selector = {message() ->\n *[key] Value\n}\n" + }, + { + "type": "Comment", + "content": "ERROR Equivalent to a MessageReference callee." + }, + { + "type": "Junk", + "annotations": [], + "content": "mixed-case-callee-selector = {Function() ->\n *[key] Value\n}\n" + }, + { + "type": "Comment", + "content": "ERROR Message attributes cannot be parameterized." + }, + { + "type": "Junk", + "annotations": [], + "content": "message-attr-callee-selector = {message.attr() ->\n *[key] Value\n}\n" + }, + { + "type": "Comment", + "content": "ERROR Term values may not be used as selectors." + }, + { + "type": "Junk", + "annotations": [], + "content": "term-callee-selector = {-term() ->\n *[key] Value\n}\n" + }, + { + "type": "Comment", + "content": "ERROR Variables cannot be parameterized." + }, + { + "type": "Junk", + "annotations": [], + "content": "variable-callee-selector = {$variable() ->\n *[key] Value\n}\n" + } + ] +} diff --git a/tests/syntax/fixtures_reference/comments.ftl b/tests/syntax/fixtures_reference/comments.ftl new file mode 100644 index 00000000..cc3246ea --- /dev/null +++ b/tests/syntax/fixtures_reference/comments.ftl @@ -0,0 +1,15 @@ +# Standalone Comment + +# Message Comment +foo = Foo + +# Term Comment +# with a blank last line. +# +-term = Term + +# Another standalone +# +# with indent +## Group Comment +### Resource Comment diff --git a/tests/syntax/fixtures_reference/comments.json b/tests/syntax/fixtures_reference/comments.json new file mode 100644 index 00000000..c28115ac --- /dev/null +++ b/tests/syntax/fixtures_reference/comments.json @@ -0,0 +1,63 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Comment", + "content": "Standalone Comment" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "foo" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Foo" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "Message Comment" + } + }, + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "term" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Term" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "Term Comment\nwith a blank last line.\n" + } + }, + { + "type": "Comment", + "content": "Another standalone\n\n with indent" + }, + { + "type": "GroupComment", + "content": "Group Comment" + }, + { + "type": "ResourceComment", + "content": "Resource Comment" + } + ] +} diff --git a/tests/syntax/fixtures_reference/cr.ftl b/tests/syntax/fixtures_reference/cr.ftl new file mode 100644 index 00000000..549c662a --- /dev/null +++ b/tests/syntax/fixtures_reference/cr.ftl @@ -0,0 +1 @@ +### This entire file uses CR as EOL. err01 = Value 01 err02 = Value 02 err03 = Value 03 Continued .title = Title err04 = { "str err05 = { $sel -> } \ No newline at end of file diff --git a/tests/syntax/fixtures_reference/cr.json b/tests/syntax/fixtures_reference/cr.json new file mode 100644 index 00000000..44eab75f --- /dev/null +++ b/tests/syntax/fixtures_reference/cr.json @@ -0,0 +1,9 @@ +{ + "type": "Resource", + "body": [ + { + "type": "ResourceComment", + "content": "This entire file uses CR as EOL.\r\rerr01 = Value 01\rerr02 = Value 02\r\rerr03 =\r\r Value 03\r Continued\r\r .title = Title\r\rerr04 = { \"str\r\rerr05 = { $sel -> }\r" + } + ] +} diff --git a/tests/syntax/fixtures_reference/crlf.ftl b/tests/syntax/fixtures_reference/crlf.ftl new file mode 100644 index 00000000..df3a02c5 --- /dev/null +++ b/tests/syntax/fixtures_reference/crlf.ftl @@ -0,0 +1,14 @@ + +key01 = Value 01 +key02 = + + Value 02 + Continued + + .title = Title + +# ERROR Unclosed StringLiteral +err03 = { "str + +# ERROR Missing newline after ->. +err04 = { $sel -> } diff --git a/tests/syntax/fixtures_reference/crlf.json b/tests/syntax/fixtures_reference/crlf.json new file mode 100644 index 00000000..58d26a77 --- /dev/null +++ b/tests/syntax/fixtures_reference/crlf.json @@ -0,0 +1,76 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key01" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value 01" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key02" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value 02\nContinued" + } + ] + }, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "title" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Title" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR Unclosed StringLiteral" + }, + { + "type": "Junk", + "annotations": [], + "content": "err03 = { \"str\r\n\r\n" + }, + { + "type": "Comment", + "content": "ERROR Missing newline after ->." + }, + { + "type": "Junk", + "annotations": [], + "content": "err04 = { $sel -> }\r\n" + } + ] +} diff --git a/tests/syntax/fixtures_reference/eof_comment.ftl b/tests/syntax/fixtures_reference/eof_comment.ftl new file mode 100644 index 00000000..cdeafd90 --- /dev/null +++ b/tests/syntax/fixtures_reference/eof_comment.ftl @@ -0,0 +1,3 @@ +### NOTE: Disable final newline insertion when editing this file. + +# No EOL \ No newline at end of file diff --git a/tests/syntax/fixtures_reference/eof_comment.json b/tests/syntax/fixtures_reference/eof_comment.json new file mode 100644 index 00000000..9483a1eb --- /dev/null +++ b/tests/syntax/fixtures_reference/eof_comment.json @@ -0,0 +1,13 @@ +{ + "type": "Resource", + "body": [ + { + "type": "ResourceComment", + "content": "NOTE: Disable final newline insertion when editing this file." + }, + { + "type": "Comment", + "content": "No EOL" + } + ] +} diff --git a/tests/syntax/fixtures_reference/eof_empty.ftl b/tests/syntax/fixtures_reference/eof_empty.ftl new file mode 100644 index 00000000..e69de29b diff --git a/tests/syntax/fixtures_reference/eof_empty.json b/tests/syntax/fixtures_reference/eof_empty.json new file mode 100644 index 00000000..b1992785 --- /dev/null +++ b/tests/syntax/fixtures_reference/eof_empty.json @@ -0,0 +1,4 @@ +{ + "type": "Resource", + "body": [] +} diff --git a/tests/syntax/fixtures_reference/eof_id.ftl b/tests/syntax/fixtures_reference/eof_id.ftl new file mode 100644 index 00000000..63fa86d6 --- /dev/null +++ b/tests/syntax/fixtures_reference/eof_id.ftl @@ -0,0 +1,3 @@ +### NOTE: Disable final newline insertion when editing this file. + +message-id \ No newline at end of file diff --git a/tests/syntax/fixtures_reference/eof_id.json b/tests/syntax/fixtures_reference/eof_id.json new file mode 100644 index 00000000..93693f94 --- /dev/null +++ b/tests/syntax/fixtures_reference/eof_id.json @@ -0,0 +1,14 @@ +{ + "type": "Resource", + "body": [ + { + "type": "ResourceComment", + "content": "NOTE: Disable final newline insertion when editing this file." + }, + { + "type": "Junk", + "annotations": [], + "content": "message-id" + } + ] +} diff --git a/tests/syntax/fixtures_reference/eof_id_equals.ftl b/tests/syntax/fixtures_reference/eof_id_equals.ftl new file mode 100644 index 00000000..7d0d953a --- /dev/null +++ b/tests/syntax/fixtures_reference/eof_id_equals.ftl @@ -0,0 +1,3 @@ +### NOTE: Disable final newline insertion when editing this file. + +message-id = \ No newline at end of file diff --git a/tests/syntax/fixtures_reference/eof_id_equals.json b/tests/syntax/fixtures_reference/eof_id_equals.json new file mode 100644 index 00000000..eefd3285 --- /dev/null +++ b/tests/syntax/fixtures_reference/eof_id_equals.json @@ -0,0 +1,14 @@ +{ + "type": "Resource", + "body": [ + { + "type": "ResourceComment", + "content": "NOTE: Disable final newline insertion when editing this file." + }, + { + "type": "Junk", + "annotations": [], + "content": "message-id =" + } + ] +} diff --git a/tests/syntax/fixtures_reference/eof_junk.ftl b/tests/syntax/fixtures_reference/eof_junk.ftl new file mode 100644 index 00000000..dbafd3a3 --- /dev/null +++ b/tests/syntax/fixtures_reference/eof_junk.ftl @@ -0,0 +1,3 @@ +### NOTE: Disable final newline insertion when editing this file. + +000 \ No newline at end of file diff --git a/tests/syntax/fixtures_reference/eof_junk.json b/tests/syntax/fixtures_reference/eof_junk.json new file mode 100644 index 00000000..d7d12824 --- /dev/null +++ b/tests/syntax/fixtures_reference/eof_junk.json @@ -0,0 +1,14 @@ +{ + "type": "Resource", + "body": [ + { + "type": "ResourceComment", + "content": "NOTE: Disable final newline insertion when editing this file." + }, + { + "type": "Junk", + "annotations": [], + "content": "000" + } + ] +} diff --git a/tests/syntax/fixtures_reference/eof_value.ftl b/tests/syntax/fixtures_reference/eof_value.ftl new file mode 100644 index 00000000..0d255c57 --- /dev/null +++ b/tests/syntax/fixtures_reference/eof_value.ftl @@ -0,0 +1,3 @@ +### NOTE: Disable final newline insertion when editing this file. + +no-eol = No EOL \ No newline at end of file diff --git a/tests/syntax/fixtures_reference/eof_value.json b/tests/syntax/fixtures_reference/eof_value.json new file mode 100644 index 00000000..502435fc --- /dev/null +++ b/tests/syntax/fixtures_reference/eof_value.json @@ -0,0 +1,27 @@ +{ + "type": "Resource", + "body": [ + { + "type": "ResourceComment", + "content": "NOTE: Disable final newline insertion when editing this file." + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "no-eol" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "No EOL" + } + ] + }, + "attributes": [], + "comment": null + } + ] +} diff --git a/tests/syntax/fixtures_reference/escaped_characters.ftl b/tests/syntax/fixtures_reference/escaped_characters.ftl new file mode 100644 index 00000000..ec862320 --- /dev/null +++ b/tests/syntax/fixtures_reference/escaped_characters.ftl @@ -0,0 +1,34 @@ +## Literal text +text-backslash-one = Value with \ a backslash +text-backslash-two = Value with \\ two backslashes +text-backslash-brace = Value with \{placeable} +text-backslash-u = \u0041 +text-backslash-backslash-u = \\u0041 + +## String literals +quote-in-string = {"\""} +backslash-in-string = {"\\"} +# ERROR Mismatched quote +mismatched-quote = {"\\""} +# ERROR Unknown escape +unknown-escape = {"\x"} + +## Unicode escapes +string-unicode-4digits = {"\u0041"} +escape-unicode-4digits = {"\\u0041"} +string-unicode-6digits = {"\U01F602"} +escape-unicode-6digits = {"\\U01F602"} + +# OK The trailing "00" is part of the literal value. +string-too-many-4digits = {"\u004100"} +# OK The trailing "00" is part of the literal value. +string-too-many-6digits = {"\U01F60200"} + +# ERROR Too few hex digits after \u. +string-too-few-4digits = {"\u41"} +# ERROR Too few hex digits after \U. +string-too-few-6digits = {"\U1F602"} + +## Literal braces +brace-open = An opening {"{"} brace. +brace-close = A closing {"}"} brace. diff --git a/tests/syntax/fixtures_reference/escaped_characters.json b/tests/syntax/fixtures_reference/escaped_characters.json new file mode 100644 index 00000000..a3220996 --- /dev/null +++ b/tests/syntax/fixtures_reference/escaped_characters.json @@ -0,0 +1,399 @@ +{ + "type": "Resource", + "body": [ + { + "type": "GroupComment", + "content": "Literal text" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "text-backslash-one" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value with \\ a backslash" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "text-backslash-two" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value with \\\\ two backslashes" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "text-backslash-brace" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value with \\" + }, + { + "type": "Placeable", + "expression": { + "type": "MessageReference", + "id": { + "type": "Identifier", + "name": "placeable" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "text-backslash-u" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "\\u0041" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "text-backslash-backslash-u" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "\\\\u0041" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "GroupComment", + "content": "String literals" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "quote-in-string" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\\"", + "value": "\"" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "backslash-in-string" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\\\", + "value": "\\" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR Mismatched quote" + }, + { + "type": "Junk", + "annotations": [], + "content": "mismatched-quote = {\"\\\\\"\"}\n" + }, + { + "type": "Comment", + "content": "ERROR Unknown escape" + }, + { + "type": "Junk", + "annotations": [], + "content": "unknown-escape = {\"\\x\"}\n\n" + }, + { + "type": "GroupComment", + "content": "Unicode escapes" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "string-unicode-4digits" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\u0041", + "value": "A" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "escape-unicode-4digits" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\\\u0041", + "value": "\\u0041" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "string-unicode-6digits" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\U01F602", + "value": "😂" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "escape-unicode-6digits" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\\\U01F602", + "value": "\\U01F602" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "string-too-many-4digits" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\u004100", + "value": "A00" + } + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "OK The trailing \"00\" is part of the literal value." + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "string-too-many-6digits" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "\\U01F60200", + "value": "😂00" + } + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "OK The trailing \"00\" is part of the literal value." + } + }, + { + "type": "Comment", + "content": "ERROR Too few hex digits after \\u." + }, + { + "type": "Junk", + "annotations": [], + "content": "string-too-few-4digits = {\"\\u41\"}\n" + }, + { + "type": "Comment", + "content": "ERROR Too few hex digits after \\U." + }, + { + "type": "Junk", + "annotations": [], + "content": "string-too-few-6digits = {\"\\U1F602\"}\n\n" + }, + { + "type": "GroupComment", + "content": "Literal braces" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "brace-open" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "An opening " + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "{", + "value": "{" + } + }, + { + "type": "TextElement", + "value": " brace." + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "brace-close" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "A closing " + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "}", + "value": "}" + } + }, + { + "type": "TextElement", + "value": " brace." + } + ] + }, + "attributes": [], + "comment": null + } + ] +} diff --git a/tests/syntax/fixtures_reference/junk.ftl b/tests/syntax/fixtures_reference/junk.ftl new file mode 100644 index 00000000..b0b0c5f3 --- /dev/null +++ b/tests/syntax/fixtures_reference/junk.ftl @@ -0,0 +1,21 @@ +## Two adjacent Junks. +err01 = {1x} +err02 = {2x} + +# A single Junk. +err03 = {1x +2 + +# A single Junk. +ą=Invalid identifier +ć=Another one + +# The COMMENT ends this junk. +err04 = { +# COMMENT + +# The COMMENT ends this junk. +# The closing brace is a separate Junk. +err04 = { +# COMMENT +} diff --git a/tests/syntax/fixtures_reference/junk.json b/tests/syntax/fixtures_reference/junk.json new file mode 100644 index 00000000..15e62a4d --- /dev/null +++ b/tests/syntax/fixtures_reference/junk.json @@ -0,0 +1,68 @@ +{ + "type": "Resource", + "body": [ + { + "type": "GroupComment", + "content": "Two adjacent Junks." + }, + { + "type": "Junk", + "annotations": [], + "content": "err01 = {1x}\n" + }, + { + "type": "Junk", + "annotations": [], + "content": "err02 = {2x}\n\n" + }, + { + "type": "Comment", + "content": "A single Junk." + }, + { + "type": "Junk", + "annotations": [], + "content": "err03 = {1x\n2\n\n" + }, + { + "type": "Comment", + "content": "A single Junk." + }, + { + "type": "Junk", + "annotations": [], + "content": "ą=Invalid identifier\nć=Another one\n\n" + }, + { + "type": "Comment", + "content": "The COMMENT ends this junk." + }, + { + "type": "Junk", + "annotations": [], + "content": "err04 = {\n" + }, + { + "type": "Comment", + "content": "COMMENT" + }, + { + "type": "Comment", + "content": "The COMMENT ends this junk.\nThe closing brace is a separate Junk." + }, + { + "type": "Junk", + "annotations": [], + "content": "err04 = {\n" + }, + { + "type": "Comment", + "content": "COMMENT" + }, + { + "type": "Junk", + "annotations": [], + "content": "}\n" + } + ] +} diff --git a/tests/syntax/fixtures_reference/leading_dots.ftl b/tests/syntax/fixtures_reference/leading_dots.ftl new file mode 100644 index 00000000..0b9d6693 --- /dev/null +++ b/tests/syntax/fixtures_reference/leading_dots.ftl @@ -0,0 +1,76 @@ +key01 = .Value +key02 = …Value +key03 = {"."}Value +key04 = + {"."}Value + +key05 = Value + {"."}Continued + +key06 = .Value + {"."}Continued + +# MESSAGE (value = "Value", attributes = []) +# JUNK (attr .Continued" must have a value) +key07 = Value + .Continued + +# JUNK (attr .Value must have a value) +key08 = + .Value + +# JUNK (attr .Value must have a value) +key09 = + .Value + Continued + +key10 = + .Value = which is an attribute + Continued + +key11 = + {"."}Value = which looks like an attribute + Continued + +key12 = + .accesskey = + A + +key13 = + .attribute = .Value + +key14 = + .attribute = + {"."}Value + +key15 = + { 1 -> + [one] .Value + *[other] + {"."}Value + } + +# JUNK (variant must have a value) +key16 = + { 1 -> + *[one] + .Value + } + +# JUNK (unclosed placeable) +key17 = + { 1 -> + *[one] Value + .Continued + } + +# JUNK (attr .Value must have a value) +key18 = +.Value + +key19 = +.attribute = Value + Continued + +key20 = +{"."}Value diff --git a/tests/syntax/fixtures_reference/leading_dots.json b/tests/syntax/fixtures_reference/leading_dots.json new file mode 100644 index 00000000..a185d489 --- /dev/null +++ b/tests/syntax/fixtures_reference/leading_dots.json @@ -0,0 +1,487 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key01" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": ".Value" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key02" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "…Value" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key03" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": ".", + "value": "." + } + }, + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key04" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": ".", + "value": "." + } + }, + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key05" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value\n" + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": ".", + "value": "." + } + }, + { + "type": "TextElement", + "value": "Continued" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key06" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": ".Value\n" + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": ".", + "value": "." + } + }, + { + "type": "TextElement", + "value": "Continued" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key07" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "MESSAGE (value = \"Value\", attributes = [])\nJUNK (attr .Continued\" must have a value)" + } + }, + { + "type": "Junk", + "annotations": [], + "content": " .Continued\n\n" + }, + { + "type": "Comment", + "content": "JUNK (attr .Value must have a value)" + }, + { + "type": "Junk", + "annotations": [], + "content": "key08 =\n .Value\n\n" + }, + { + "type": "Comment", + "content": "JUNK (attr .Value must have a value)" + }, + { + "type": "Junk", + "annotations": [], + "content": "key09 =\n .Value\n Continued\n\n" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key10" + }, + "value": null, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "Value" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "which is an attribute\nContinued" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key11" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": ".", + "value": "." + } + }, + { + "type": "TextElement", + "value": "Value = which looks like an attribute\nContinued" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key12" + }, + "value": null, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "accesskey" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "A" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key13" + }, + "value": null, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attribute" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": ".Value" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key14" + }, + "value": null, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attribute" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": ".", + "value": "." + } + }, + { + "type": "TextElement", + "value": "Value" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key15" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "NumberLiteral", + "value": "1" + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "one" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": ".Value" + } + ] + }, + "default": false + }, + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "other" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": ".", + "value": "." + } + }, + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "JUNK (variant must have a value)" + }, + { + "type": "Junk", + "annotations": [], + "content": "key16 =\n { 1 ->\n *[one]\n .Value\n }\n\n" + }, + { + "type": "Comment", + "content": "JUNK (unclosed placeable)" + }, + { + "type": "Junk", + "annotations": [], + "content": "key17 =\n { 1 ->\n *[one] Value\n .Continued\n }\n\n" + }, + { + "type": "Comment", + "content": "JUNK (attr .Value must have a value)" + }, + { + "type": "Junk", + "annotations": [], + "content": "key18 =\n.Value\n\n" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key19" + }, + "value": null, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attribute" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value\nContinued" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key20" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": ".", + "value": "." + } + }, + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [], + "comment": null + } + ] +} diff --git a/tests/syntax/fixtures_reference/literal_expressions.ftl b/tests/syntax/fixtures_reference/literal_expressions.ftl new file mode 100644 index 00000000..937b8a17 --- /dev/null +++ b/tests/syntax/fixtures_reference/literal_expressions.ftl @@ -0,0 +1,3 @@ +string-expression = {"abc"} +number-expression = {123} +number-expression = {-3.14} diff --git a/tests/syntax/fixtures_reference/literal_expressions.json b/tests/syntax/fixtures_reference/literal_expressions.json new file mode 100644 index 00000000..da73e131 --- /dev/null +++ b/tests/syntax/fixtures_reference/literal_expressions.json @@ -0,0 +1,69 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "string-expression" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "abc", + "value": "abc" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "number-expression" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "NumberLiteral", + "value": "123" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "number-expression" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "NumberLiteral", + "value": "-3.14" + } + } + ] + }, + "attributes": [], + "comment": null + } + ] +} diff --git a/tests/syntax/fixtures_reference/member_expressions.ftl b/tests/syntax/fixtures_reference/member_expressions.ftl new file mode 100644 index 00000000..b4a93922 --- /dev/null +++ b/tests/syntax/fixtures_reference/member_expressions.ftl @@ -0,0 +1,28 @@ +## Member expressions in placeables. + +message-attribute-expression-placeable = {msg.attr} +term-variant-expression-placeable = {-term[case]} + +# ERROR Message values cannot be VariantLists +message-variant-expression-placeable = {msg[case]} +# ERROR Term attributes may not be used for interpolation. +term-attribute-expression-placeable = {-term.attr} + +## Member expressions in selectors. + +term-attribute-expression-selector = {-term.attr -> + *[key] Value +} + +# ERROR Message attributes may not be used as selector. +message-attribute-expression-selector = {msg.attr -> + *[key] Value +} +# ERROR Term values may not be used as selector. +term-variant-expression-selector = {-term[case] -> + *[key] Value +} +# ERROR Message values cannot be VariantLists +message-variant-expression-selector = {msg[case] -> + *[key] Value +} diff --git a/tests/syntax/fixtures_reference/member_expressions.json b/tests/syntax/fixtures_reference/member_expressions.json new file mode 100644 index 00000000..f6890e56 --- /dev/null +++ b/tests/syntax/fixtures_reference/member_expressions.json @@ -0,0 +1,173 @@ +{ + "type": "Resource", + "body": [ + { + "type": "GroupComment", + "content": "Member expressions in placeables." + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "message-attribute-expression-placeable" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "AttributeExpression", + "ref": { + "type": "MessageReference", + "id": { + "type": "Identifier", + "name": "msg" + } + }, + "name": { + "type": "Identifier", + "name": "attr" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "term-variant-expression-placeable" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "VariantExpression", + "ref": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + }, + "key": { + "type": "Identifier", + "name": "case" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR Message values cannot be VariantLists" + }, + { + "type": "Junk", + "annotations": [], + "content": "message-variant-expression-placeable = {msg[case]}\n" + }, + { + "type": "Comment", + "content": "ERROR Term attributes may not be used for interpolation." + }, + { + "type": "Junk", + "annotations": [], + "content": "term-attribute-expression-placeable = {-term.attr}\n\n" + }, + { + "type": "GroupComment", + "content": "Member expressions in selectors." + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "term-attribute-expression-selector" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "AttributeExpression", + "ref": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + }, + "name": { + "type": "Identifier", + "name": "attr" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR Message attributes may not be used as selector." + }, + { + "type": "Junk", + "annotations": [], + "content": "message-attribute-expression-selector = {msg.attr ->\n *[key] Value\n}\n" + }, + { + "type": "Comment", + "content": "ERROR Term values may not be used as selector." + }, + { + "type": "Junk", + "annotations": [], + "content": "term-variant-expression-selector = {-term[case] ->\n *[key] Value\n}\n" + }, + { + "type": "Comment", + "content": "ERROR Message values cannot be VariantLists" + }, + { + "type": "Junk", + "annotations": [], + "content": "message-variant-expression-selector = {msg[case] ->\n *[key] Value\n}\n" + } + ] +} diff --git a/tests/syntax/fixtures_reference/messages.ftl b/tests/syntax/fixtures_reference/messages.ftl new file mode 100644 index 00000000..00b4ab46 --- /dev/null +++ b/tests/syntax/fixtures_reference/messages.ftl @@ -0,0 +1,29 @@ +key01 = Value + +key02 = Value + .attr = Attribute + +key02 = Value + .attr1 = Attribute 1 + .attr2 = Attribute 2 + +key03 = + .attr = Attribute + +key04 = + .attr1 = Attribute 1 + .attr2 = Attribute 2 + +# < whitespace > +key05 = + .attr1 = Attribute 1 + +key06 = {""} + +# JUNK Missing value +key07 = + +# JUNK Missing = +key08 + +KEY09 = Value 09 diff --git a/tests/syntax/fixtures_reference/messages.json b/tests/syntax/fixtures_reference/messages.json new file mode 100644 index 00000000..cdbe5c93 --- /dev/null +++ b/tests/syntax/fixtures_reference/messages.json @@ -0,0 +1,267 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key01" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key02" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attr" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Attribute" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key02" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attr1" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Attribute 1" + } + ] + } + }, + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attr2" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Attribute 2" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key03" + }, + "value": null, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attr" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Attribute" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key04" + }, + "value": null, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attr1" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Attribute 1" + } + ] + } + }, + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attr2" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Attribute 2" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key05" + }, + "value": null, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attr1" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Attribute 1" + } + ] + } + } + ], + "comment": { + "type": "Comment", + "content": " < whitespace >" + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key06" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "", + "value": "" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "JUNK Missing value" + }, + { + "type": "Junk", + "annotations": [], + "content": "key07 =\n\n" + }, + { + "type": "Comment", + "content": "JUNK Missing =" + }, + { + "type": "Junk", + "annotations": [], + "content": "key08\n\n" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "KEY09" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value 09" + } + ] + }, + "attributes": [], + "comment": null + } + ] +} diff --git a/tests/syntax/fixtures_reference/mixed_entries.ftl b/tests/syntax/fixtures_reference/mixed_entries.ftl new file mode 100644 index 00000000..99cc023d --- /dev/null +++ b/tests/syntax/fixtures_reference/mixed_entries.ftl @@ -0,0 +1,24 @@ +# License Comment + +### Resource Comment + +-brand-name = Aurora + +## Group Comment + +key01 = + .attr = Attribute + +ą=Invalid identifier +ć=Another one + +# Message Comment +key02 = Value + +# Standalone Comment + .attr = Dangling attribute + +# There are 5 spaces on the line between key03 and key04. +key03 = Value 03 + +key04 = Value 04 diff --git a/tests/syntax/fixtures_reference/mixed_entries.json b/tests/syntax/fixtures_reference/mixed_entries.json new file mode 100644 index 00000000..a9dc501f --- /dev/null +++ b/tests/syntax/fixtures_reference/mixed_entries.json @@ -0,0 +1,136 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Comment", + "content": "License Comment" + }, + { + "type": "ResourceComment", + "content": "Resource Comment" + }, + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "brand-name" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Aurora" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "GroupComment", + "content": "Group Comment" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key01" + }, + "value": null, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attr" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Attribute" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Junk", + "annotations": [], + "content": "ą=Invalid identifier\nć=Another one\n\n" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key02" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "Message Comment" + } + }, + { + "type": "Comment", + "content": "Standalone Comment" + }, + { + "type": "Junk", + "annotations": [], + "content": " .attr = Dangling attribute\n\n" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key03" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value 03" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "There are 5 spaces on the line between key03 and key04." + } + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key04" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value 04" + } + ] + }, + "attributes": [], + "comment": null + } + ] +} diff --git a/tests/syntax/fixtures_reference/multiline_values.ftl b/tests/syntax/fixtures_reference/multiline_values.ftl new file mode 100644 index 00000000..e3739bb5 --- /dev/null +++ b/tests/syntax/fixtures_reference/multiline_values.ftl @@ -0,0 +1,60 @@ +key01 = A multiline value + continued on the next line + + and also down here. + +key02 = + A multiline value starting + on a new line. + +key03 = + .attr = A multiline attribute value + continued on the next line + + and also down here. + +key04 = + .attr = + A multiline attribute value + staring on a new line + +key05 = + + A multiline value with non-standard + + indentation. + +key06 = + A multiline value with {"placeables"} + {"at"} the beginning and the end + {"of lines"}{"."} + +key07 = + {"A multiline value"} starting and ending {"with a placeable"} + +key08 = Leading and trailing whitespace. + +key09 = zero + three + two + one + zero + +key10 = + two + zero + four + +key11 = + + + two + zero + +key12 = +{"."} + four + +key13 = + four +{"."} diff --git a/tests/syntax/fixtures_reference/multiline_values.json b/tests/syntax/fixtures_reference/multiline_values.json new file mode 100644 index 00000000..4d3dd033 --- /dev/null +++ b/tests/syntax/fixtures_reference/multiline_values.json @@ -0,0 +1,329 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key01" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "A multiline value\ncontinued on the next line\n\nand also down here." + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key02" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "A multiline value starting\non a new line." + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key03" + }, + "value": null, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attr" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "A multiline attribute value\ncontinued on the next line\n\nand also down here." + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key04" + }, + "value": null, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attr" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "A multiline attribute value\nstaring on a new line" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key05" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "A multiline value with non-standard\n\n indentation." + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key06" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "A multiline value with " + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "placeables", + "value": "placeables" + } + }, + { + "type": "TextElement", + "value": "\n" + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "at", + "value": "at" + } + }, + { + "type": "TextElement", + "value": " the beginning and the end\n" + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "of lines", + "value": "of lines" + } + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": ".", + "value": "." + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key07" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "A multiline value", + "value": "A multiline value" + } + }, + { + "type": "TextElement", + "value": " starting and ending " + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "with a placeable", + "value": "with a placeable" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key08" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Leading and trailing whitespace." + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key09" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "zero\n three\n two\n one\nzero" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key10" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": " two\nzero\n four" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key11" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": " two\nzero" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key12" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": ".", + "value": "." + } + }, + { + "type": "TextElement", + "value": "\n four" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key13" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": " four\n" + }, + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": ".", + "value": "." + } + } + ] + }, + "attributes": [], + "comment": null + } + ] +} diff --git a/tests/syntax/fixtures_reference/placeables.ftl b/tests/syntax/fixtures_reference/placeables.ftl new file mode 100644 index 00000000..7a1b280f --- /dev/null +++ b/tests/syntax/fixtures_reference/placeables.ftl @@ -0,0 +1,15 @@ +nested-placeable = {{{1}}} +padded-placeable = { 1 } +sparse-placeable = { { 1 } } + +# ERROR Unmatched opening brace +unmatched-open1 = { 1 + +# ERROR Unmatched opening brace +unmatched-open2 = {{ 1 } + +# ERROR Unmatched closing brace +unmatched-close1 = 1 } + +# ERROR Unmatched closing brace +unmatched-close2 = { 1 }} diff --git a/tests/syntax/fixtures_reference/placeables.json b/tests/syntax/fixtures_reference/placeables.json new file mode 100644 index 00000000..7d67d940 --- /dev/null +++ b/tests/syntax/fixtures_reference/placeables.json @@ -0,0 +1,113 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "nested-placeable" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "Placeable", + "expression": { + "type": "Placeable", + "expression": { + "type": "NumberLiteral", + "value": "1" + } + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "padded-placeable" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "NumberLiteral", + "value": "1" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "sparse-placeable" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "Placeable", + "expression": { + "type": "NumberLiteral", + "value": "1" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR Unmatched opening brace" + }, + { + "type": "Junk", + "annotations": [], + "content": "unmatched-open1 = { 1\n\n" + }, + { + "type": "Comment", + "content": "ERROR Unmatched opening brace" + }, + { + "type": "Junk", + "annotations": [], + "content": "unmatched-open2 = {{ 1 }\n\n" + }, + { + "type": "Comment", + "content": "ERROR Unmatched closing brace" + }, + { + "type": "Junk", + "annotations": [], + "content": "unmatched-close1 = 1 }\n\n" + }, + { + "type": "Comment", + "content": "ERROR Unmatched closing brace" + }, + { + "type": "Junk", + "annotations": [], + "content": "unmatched-close2 = { 1 }}\n" + } + ] +} diff --git a/tests/syntax/fixtures_reference/reference_expressions.ftl b/tests/syntax/fixtures_reference/reference_expressions.ftl new file mode 100644 index 00000000..9c2e9c54 --- /dev/null +++ b/tests/syntax/fixtures_reference/reference_expressions.ftl @@ -0,0 +1,28 @@ +## Reference expressions in placeables. + +message-reference-placeable = {msg} +term-reference-placeable = {-term} +variable-reference-placeable = {$var} + +# ERROR Function references are invalid outside of call expressions. +function-reference-placeable = {FUN} + + +## Reference expressions in selectors. + +variable-reference-selector = {$var -> + *[key] Value +} + +# ERROR Message values may not be used as selectors. +message-reference-selector = {msg -> + *[key] Value +} +# ERROR Term values may not be used as selectors. +term-reference-selector = {-term -> + *[key] Value +} +# ERROR Function references are invalid outside of call expressions. +function-expression-selector = {FUN -> + *[key] Value +} diff --git a/tests/syntax/fixtures_reference/reference_expressions.json b/tests/syntax/fixtures_reference/reference_expressions.json new file mode 100644 index 00000000..65c9d4cc --- /dev/null +++ b/tests/syntax/fixtures_reference/reference_expressions.json @@ -0,0 +1,185 @@ +{ + "type": "Resource", + "body": [ + { + "type": "GroupComment", + "content": "Reference expressions in placeables." + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "message-reference-placeable" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "MessageReference", + "id": { + "type": "Identifier", + "name": "msg" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "term-reference-placeable" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "variable-reference-placeable" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "var" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "function-reference-placeable" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "MessageReference", + "id": { + "type": "Identifier", + "name": "FUN" + } + } + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "ERROR Function references are invalid outside of call expressions." + } + }, + { + "type": "GroupComment", + "content": "Reference expressions in selectors." + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "variable-reference-selector" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "var" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR Message values may not be used as selectors." + }, + { + "type": "Junk", + "annotations": [], + "content": "message-reference-selector = {msg ->\n *[key] Value\n}\n" + }, + { + "type": "Comment", + "content": "ERROR Term values may not be used as selectors." + }, + { + "type": "Junk", + "annotations": [], + "content": "term-reference-selector = {-term ->\n *[key] Value\n}\n" + }, + { + "type": "Comment", + "content": "ERROR Function references are invalid outside of call expressions." + }, + { + "type": "Junk", + "annotations": [], + "content": "function-expression-selector = {FUN ->\n *[key] Value\n}\n" + } + ] +} diff --git a/tests/syntax/fixtures_reference/select_expressions.ftl b/tests/syntax/fixtures_reference/select_expressions.ftl new file mode 100644 index 00000000..7a1fb820 --- /dev/null +++ b/tests/syntax/fixtures_reference/select_expressions.ftl @@ -0,0 +1,53 @@ +new-messages = + { BUILTIN() -> + [0] Zero + *[other] {""}Other + } + +valid-selector-term-attribute = + { -term.case -> + *[key] value + } + +# ERROR +invalid-selector-term-value = + { -term -> + *[key] value + } + +# ERROR +invalid-selector-term-variant = + { -term[case] -> + *[key] value + } + +# ERROR +invalid-selector-term-call = + { -term(case: "nominative") -> + *[key] value + } + +empty-variant = + { 1 -> + *[one] {""} + } + +nested-select = + { 1 -> + *[one] { 2 -> + *[two] Value + } + } + +# ERROR VariantLists cannot be Variant values. +nested-variant-list = + { 1 -> + *[one] { + *[two] Value + } + } + +# ERROR Missing line end after variant list +missing-line-end = + { 1 -> + *[one] One} diff --git a/tests/syntax/fixtures_reference/select_expressions.json b/tests/syntax/fixtures_reference/select_expressions.json new file mode 100644 index 00000000..a3dc5730 --- /dev/null +++ b/tests/syntax/fixtures_reference/select_expressions.json @@ -0,0 +1,294 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "new-messages" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "CallExpression", + "callee": { + "type": "FunctionReference", + "id": { + "type": "Identifier", + "name": "BUILTIN" + } + }, + "positional": [], + "named": [] + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "NumberLiteral", + "value": "0" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Zero" + } + ] + }, + "default": false + }, + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "other" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "", + "value": "" + } + }, + { + "type": "TextElement", + "value": "Other" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "valid-selector-term-attribute" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "AttributeExpression", + "ref": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + }, + "name": { + "type": "Identifier", + "name": "case" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR" + }, + { + "type": "Junk", + "annotations": [], + "content": "invalid-selector-term-value =\n { -term ->\n *[key] value\n }\n\n" + }, + { + "type": "Comment", + "content": "ERROR" + }, + { + "type": "Junk", + "annotations": [], + "content": "invalid-selector-term-variant =\n { -term[case] ->\n *[key] value\n }\n\n" + }, + { + "type": "Comment", + "content": "ERROR" + }, + { + "type": "Junk", + "annotations": [], + "content": "invalid-selector-term-call =\n { -term(case: \"nominative\") ->\n *[key] value\n }\n\n" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "empty-variant" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "NumberLiteral", + "value": "1" + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "one" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "", + "value": "" + } + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "nested-select" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "NumberLiteral", + "value": "1" + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "one" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "NumberLiteral", + "value": "2" + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "two" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR VariantLists cannot be Variant values." + }, + { + "type": "Junk", + "annotations": [], + "content": "nested-variant-list =\n { 1 ->\n *[one] {\n *[two] Value\n }\n }\n\n" + }, + { + "type": "Comment", + "content": "ERROR Missing line end after variant list" + }, + { + "type": "Junk", + "annotations": [], + "content": "missing-line-end =\n { 1 ->\n *[one] One}\n" + } + ] +} diff --git a/tests/syntax/fixtures_reference/select_indent.ftl b/tests/syntax/fixtures_reference/select_indent.ftl new file mode 100644 index 00000000..6c13076b --- /dev/null +++ b/tests/syntax/fixtures_reference/select_indent.ftl @@ -0,0 +1,95 @@ +select-1tbs-inline = { $selector -> + *[key] Value +} + +select-1tbs-newline = { +$selector -> + *[key] Value +} + +select-1tbs-indent = { + $selector -> + *[key] Value +} + +select-allman-inline = +{ $selector -> + *[key] Value +} + +select-allman-newline = +{ +$selector -> + *[key] Value +} + +select-allman-indent = +{ + $selector -> + *[key] Value +} + +select-gnu-inline = + { $selector -> + *[key] Value + } + +select-gnu-newline = + { +$selector -> + *[key] Value + } + +select-gnu-indent = + { + $selector -> + *[key] Value + } + +select-no-indent = +{ +$selector -> +*[key] Value +[other] Other +} + +select-no-indent-multiline = +{ +$selector -> +*[key] Value + Continued +[other] + Other + Multiline +} + +# ERROR (Multiline text must be indented) +select-no-indent-multiline = { $selector -> + *[key] Value +Continued without indent. +} + +select-flat = +{ +$selector +-> +*[ +key +] Value +[ +other +] Other +} + +# Each line ends with 5 spaces. +select-flat-with-trailing-spaces = +{ +$selector +-> +*[ +key +] Value +[ +other +] Other +} diff --git a/tests/syntax/fixtures_reference/select_indent.json b/tests/syntax/fixtures_reference/select_indent.json new file mode 100644 index 00000000..d8c0fa88 --- /dev/null +++ b/tests/syntax/fixtures_reference/select_indent.json @@ -0,0 +1,688 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "select-1tbs-inline" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "selector" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "select-1tbs-newline" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "selector" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "select-1tbs-indent" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "selector" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "select-allman-inline" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "selector" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "select-allman-newline" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "selector" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "select-allman-indent" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "selector" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "select-gnu-inline" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "selector" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "select-gnu-newline" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "selector" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "select-gnu-indent" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "selector" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "select-no-indent" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "selector" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + }, + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "other" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Other" + } + ] + }, + "default": false + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "select-no-indent-multiline" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "selector" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value\nContinued" + } + ] + }, + "default": true + }, + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "other" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Other\nMultiline" + } + ] + }, + "default": false + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR (Multiline text must be indented)" + }, + { + "type": "Junk", + "annotations": [], + "content": "select-no-indent-multiline = { $selector ->\n *[key] Value\n" + }, + { + "type": "Junk", + "annotations": [], + "content": "Continued without indent.\n}\n\n" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "select-flat" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "selector" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + }, + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "other" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Other" + } + ] + }, + "default": false + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "select-flat-with-trailing-spaces" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "selector" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + }, + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "other" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Other" + } + ] + }, + "default": false + } + ] + } + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "Each line ends with 5 spaces." + } + } + ] +} diff --git a/tests/syntax/fixtures_reference/sparse_entries.ftl b/tests/syntax/fixtures_reference/sparse_entries.ftl new file mode 100644 index 00000000..67920b2c --- /dev/null +++ b/tests/syntax/fixtures_reference/sparse_entries.ftl @@ -0,0 +1,39 @@ +key01 = + + + Value + +key02 = + + + .attr = Attribute + + +key03 = + Value + Continued + + + Over multiple + Lines + + + + .attr = Attribute + + +key05 = Value + +key06 = { 1 -> + + + [one] One + + + + + *[two] Two + + + + } diff --git a/tests/syntax/fixtures_reference/sparse_entries.json b/tests/syntax/fixtures_reference/sparse_entries.json new file mode 100644 index 00000000..0c4a3686 --- /dev/null +++ b/tests/syntax/fixtures_reference/sparse_entries.json @@ -0,0 +1,163 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key01" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key02" + }, + "value": null, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attr" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Attribute" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key03" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value\nContinued\n\n\nOver multiple\nLines" + } + ] + }, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attr" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Attribute" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key05" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key06" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "NumberLiteral", + "value": "1" + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "one" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "One" + } + ] + }, + "default": false + }, + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "two" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Two" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + } + ] +} diff --git a/tests/syntax/fixtures_reference/tab.ftl b/tests/syntax/fixtures_reference/tab.ftl new file mode 100644 index 00000000..4b23ad87 --- /dev/null +++ b/tests/syntax/fixtures_reference/tab.ftl @@ -0,0 +1,14 @@ +# OK (tab after = is part of the value) +key01 = Value 01 + +# Error (tab before =) +key02 = Value 02 + +# Error (tab is not a valid indent) +key03 = + This line isn't properly indented. + +# Partial Error (tab is not a valid indent) +key04 = + This line is indented by 4 spaces, + whereas this line by 1 tab. diff --git a/tests/syntax/fixtures_reference/tab.json b/tests/syntax/fixtures_reference/tab.json new file mode 100644 index 00000000..714eb947 --- /dev/null +++ b/tests/syntax/fixtures_reference/tab.json @@ -0,0 +1,70 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key01" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "\tValue 01" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "OK (tab after = is part of the value)" + } + }, + { + "type": "Comment", + "content": "Error (tab before =)" + }, + { + "type": "Junk", + "annotations": [], + "content": "key02\t= Value 02\n\n" + }, + { + "type": "Comment", + "content": "Error (tab is not a valid indent)" + }, + { + "type": "Junk", + "annotations": [], + "content": "key03 =\n\tThis line isn't properly indented.\n\n" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key04" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "This line is indented by 4 spaces," + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "Partial Error (tab is not a valid indent)" + } + }, + { + "type": "Junk", + "annotations": [], + "content": "\twhereas this line by 1 tab.\n" + } + ] +} diff --git a/tests/syntax/fixtures_reference/term_parameters.ftl b/tests/syntax/fixtures_reference/term_parameters.ftl new file mode 100644 index 00000000..61442361 --- /dev/null +++ b/tests/syntax/fixtures_reference/term_parameters.ftl @@ -0,0 +1,8 @@ +-term = { $arg -> + *[key] Value +} + +key01 = { -term } +key02 = { -term() } +key03 = { -term(arg: 1) } +key04 = { -term("positional", narg1: 1, narg2: 2) } diff --git a/tests/syntax/fixtures_reference/term_parameters.json b/tests/syntax/fixtures_reference/term_parameters.json new file mode 100644 index 00000000..f9f09613 --- /dev/null +++ b/tests/syntax/fixtures_reference/term_parameters.json @@ -0,0 +1,203 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "term" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "arg" + } + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key01" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key02" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + }, + "positional": [], + "named": [] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key03" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + }, + "positional": [], + "named": [ + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "arg" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key04" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "CallExpression", + "callee": { + "type": "TermReference", + "id": { + "type": "Identifier", + "name": "term" + } + }, + "positional": [ + { + "type": "StringLiteral", + "raw": "positional", + "value": "positional" + } + ], + "named": [ + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "narg1" + }, + "value": { + "type": "NumberLiteral", + "value": "1" + } + }, + { + "type": "NamedArgument", + "name": { + "type": "Identifier", + "name": "narg2" + }, + "value": { + "type": "NumberLiteral", + "value": "2" + } + } + ] + } + } + ] + }, + "attributes": [], + "comment": null + } + ] +} diff --git a/tests/syntax/fixtures_reference/terms.ftl b/tests/syntax/fixtures_reference/terms.ftl new file mode 100644 index 00000000..b8791fcf --- /dev/null +++ b/tests/syntax/fixtures_reference/terms.ftl @@ -0,0 +1,23 @@ +-term01 = Value + .attr = Attribute + +-term02 = {""} + +# JUNK Missing value +-term03 = + .attr = Attribute + +# JUNK Missing value +# < whitespace > +-term04 = + .attr1 = Attribute 1 + +# JUNK Missing value +-term05 = + +# JUNK Missing value +# < whitespace > +-term06 = + +# JUNK Missing = +-term07 diff --git a/tests/syntax/fixtures_reference/terms.json b/tests/syntax/fixtures_reference/terms.json new file mode 100644 index 00000000..2321283f --- /dev/null +++ b/tests/syntax/fixtures_reference/terms.json @@ -0,0 +1,107 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "term01" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [ + { + "type": "Attribute", + "id": { + "type": "Identifier", + "name": "attr" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Attribute" + } + ] + } + } + ], + "comment": null + }, + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "term02" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "StringLiteral", + "raw": "", + "value": "" + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "JUNK Missing value" + }, + { + "type": "Junk", + "annotations": [], + "content": "-term03 =\n .attr = Attribute\n\n" + }, + { + "type": "Comment", + "content": "JUNK Missing value\n < whitespace >" + }, + { + "type": "Junk", + "annotations": [], + "content": "-term04 = \n .attr1 = Attribute 1\n\n" + }, + { + "type": "Comment", + "content": "JUNK Missing value" + }, + { + "type": "Junk", + "annotations": [], + "content": "-term05 =\n\n" + }, + { + "type": "Comment", + "content": "JUNK Missing value\n < whitespace >" + }, + { + "type": "Junk", + "annotations": [], + "content": "-term06 = \n\n" + }, + { + "type": "Comment", + "content": "JUNK Missing =" + }, + { + "type": "Junk", + "annotations": [], + "content": "-term07\n" + } + ] +} diff --git a/tests/syntax/fixtures_reference/variables.ftl b/tests/syntax/fixtures_reference/variables.ftl new file mode 100644 index 00000000..6c343692 --- /dev/null +++ b/tests/syntax/fixtures_reference/variables.ftl @@ -0,0 +1,17 @@ +key01 = {$var} +key02 = { $var } +key03 = { + $var +} +key04 = { +$var} + + +## Errors + +# ERROR Missing variable identifier +err01 = {$} +# ERROR Double $$ +err02 = {$$var} +# ERROR Invalid first char of the identifier +err03 = {$-var} diff --git a/tests/syntax/fixtures_reference/variables.json b/tests/syntax/fixtures_reference/variables.json new file mode 100644 index 00000000..58682e5b --- /dev/null +++ b/tests/syntax/fixtures_reference/variables.json @@ -0,0 +1,132 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key01" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "var" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key02" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "var" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key03" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "var" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key04" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "VariableReference", + "id": { + "type": "Identifier", + "name": "var" + } + } + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "GroupComment", + "content": "Errors" + }, + { + "type": "Comment", + "content": "ERROR Missing variable identifier" + }, + { + "type": "Junk", + "annotations": [], + "content": "err01 = {$}\n" + }, + { + "type": "Comment", + "content": "ERROR Double $$" + }, + { + "type": "Junk", + "annotations": [], + "content": "err02 = {$$var}\n" + }, + { + "type": "Comment", + "content": "ERROR Invalid first char of the identifier" + }, + { + "type": "Junk", + "annotations": [], + "content": "err03 = {$-var}\n" + } + ] +} diff --git a/tests/syntax/fixtures_reference/variant_keys.ftl b/tests/syntax/fixtures_reference/variant_keys.ftl new file mode 100644 index 00000000..7586d524 --- /dev/null +++ b/tests/syntax/fixtures_reference/variant_keys.ftl @@ -0,0 +1,37 @@ +-simple-identifier = + { + *[key] value + } + +-identifier-surrounded-by-whitespace = + { + *[ key ] value + } + +-int-number = + { + *[1] value + } + +-float-number = + { + *[3.14] value + } + +# ERROR +-invalid-identifier = + { + *[two words] value + } + +# ERROR +-invalid-int = + { + *[1 apple] value + } + +# ERROR +-invalid-int = + { + *[3.14 apples] value + } diff --git a/tests/syntax/fixtures_reference/variant_keys.json b/tests/syntax/fixtures_reference/variant_keys.json new file mode 100644 index 00000000..cf752e43 --- /dev/null +++ b/tests/syntax/fixtures_reference/variant_keys.json @@ -0,0 +1,156 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "simple-identifier" + }, + "value": { + "type": "VariantList", + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "value" + } + ] + }, + "default": true + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "identifier-surrounded-by-whitespace" + }, + "value": { + "type": "VariantList", + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "value" + } + ] + }, + "default": true + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "int-number" + }, + "value": { + "type": "VariantList", + "variants": [ + { + "type": "Variant", + "key": { + "type": "NumberLiteral", + "value": "1" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "value" + } + ] + }, + "default": true + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "float-number" + }, + "value": { + "type": "VariantList", + "variants": [ + { + "type": "Variant", + "key": { + "type": "NumberLiteral", + "value": "3.14" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "value" + } + ] + }, + "default": true + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR" + }, + { + "type": "Junk", + "annotations": [], + "content": "-invalid-identifier =\n {\n *[two words] value\n }\n\n" + }, + { + "type": "Comment", + "content": "ERROR" + }, + { + "type": "Junk", + "annotations": [], + "content": "-invalid-int =\n {\n *[1 apple] value\n }\n\n" + }, + { + "type": "Comment", + "content": "ERROR" + }, + { + "type": "Junk", + "annotations": [], + "content": "-invalid-int =\n {\n *[3.14 apples] value\n }\n" + } + ] +} diff --git a/tests/syntax/fixtures_reference/variant_lists.ftl b/tests/syntax/fixtures_reference/variant_lists.ftl new file mode 100644 index 00000000..e5c61dd8 --- /dev/null +++ b/tests/syntax/fixtures_reference/variant_lists.ftl @@ -0,0 +1,55 @@ +-variant-list-in-term = + { + *[key] Value + } + +# ERROR Attributes of Terms must be Patterns. +-variant-list-in-term-attr = Value + .attr = + { + *[key] Value + } + +# ERROR Message values must be Patterns. +variant-list-in-message = + { + *[key] Value + } + +# ERROR Attributes of Messages must be Patterns. +variant-list-in-message-attr = Value + .attr = + { + *[key] Value + } + +# ERROR VariantLists cannot be Variant values. +-nested-variant-list-in-term = + { + *[one] { + *[two] Value + } + } + +-nested-select = + { + *[one] { 2 -> + *[two] Value + } + } + +# ERROR VariantLists cannot be Variant values. +nested-select-then-variant-list = + { + *[one] { 2 -> + *[two] { + *[three] Value + } + } + } + +# ERROR VariantLists are value types and may not appear in Placeables +variant-list-in-placeable = + A prefix here { + *[key] Value + } and a postfix here make this a Pattern. diff --git a/tests/syntax/fixtures_reference/variant_lists.json b/tests/syntax/fixtures_reference/variant_lists.json new file mode 100644 index 00000000..83f16662 --- /dev/null +++ b/tests/syntax/fixtures_reference/variant_lists.json @@ -0,0 +1,180 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "variant-list-in-term" + }, + "value": { + "type": "VariantList", + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "variant-list-in-term-attr" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "ERROR Attributes of Terms must be Patterns." + } + }, + { + "type": "Junk", + "annotations": [], + "content": " .attr =\n {\n *[key] Value\n }\n\n" + }, + { + "type": "Comment", + "content": "ERROR Message values must be Patterns." + }, + { + "type": "Junk", + "annotations": [], + "content": "variant-list-in-message =\n {\n *[key] Value\n }\n\n" + }, + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "variant-list-in-message-attr" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "ERROR Attributes of Messages must be Patterns." + } + }, + { + "type": "Junk", + "annotations": [], + "content": " .attr =\n {\n *[key] Value\n }\n\n" + }, + { + "type": "Comment", + "content": "ERROR VariantLists cannot be Variant values." + }, + { + "type": "Junk", + "annotations": [], + "content": "-nested-variant-list-in-term =\n {\n *[one] {\n *[two] Value\n }\n }\n\n" + }, + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "nested-select" + }, + "value": { + "type": "VariantList", + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "one" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "Placeable", + "expression": { + "type": "SelectExpression", + "selector": { + "type": "NumberLiteral", + "value": "2" + }, + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "two" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + } + } + ] + }, + "default": true + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Comment", + "content": "ERROR VariantLists cannot be Variant values." + }, + { + "type": "Junk", + "annotations": [], + "content": "nested-select-then-variant-list =\n {\n *[one] { 2 ->\n *[two] {\n *[three] Value\n }\n }\n }\n\n" + }, + { + "type": "Comment", + "content": "ERROR VariantLists are value types and may not appear in Placeables" + }, + { + "type": "Junk", + "annotations": [], + "content": "variant-list-in-placeable =\n A prefix here {\n *[key] Value\n } and a postfix here make this a Pattern.\n" + } + ] +} diff --git a/tests/syntax/fixtures_reference/variants_indent.ftl b/tests/syntax/fixtures_reference/variants_indent.ftl new file mode 100644 index 00000000..38f5a62e --- /dev/null +++ b/tests/syntax/fixtures_reference/variants_indent.ftl @@ -0,0 +1,19 @@ +-variants-1tbs = { + *[key] Value +} + +-variants-allman = +{ + *[key] Value +} + +-variants-gnu = + { + *[key] Value + } + +-variants-no-indent = +{ +*[key] Value +[other] Other +} diff --git a/tests/syntax/fixtures_reference/variants_indent.json b/tests/syntax/fixtures_reference/variants_indent.json new file mode 100644 index 00000000..7b987d48 --- /dev/null +++ b/tests/syntax/fixtures_reference/variants_indent.json @@ -0,0 +1,146 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "variants-1tbs" + }, + "value": { + "type": "VariantList", + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "variants-allman" + }, + "value": { + "type": "VariantList", + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "variants-gnu" + }, + "value": { + "type": "VariantList", + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + } + ] + }, + "attributes": [], + "comment": null + }, + { + "type": "Term", + "id": { + "type": "Identifier", + "name": "variants-no-indent" + }, + "value": { + "type": "VariantList", + "variants": [ + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Value" + } + ] + }, + "default": true + }, + { + "type": "Variant", + "key": { + "type": "Identifier", + "name": "other" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "Other" + } + ] + }, + "default": false + } + ] + }, + "attributes": [], + "comment": null + } + ] +} diff --git a/tests/syntax/fixtures_reference/whitespace_in_value.ftl b/tests/syntax/fixtures_reference/whitespace_in_value.ftl new file mode 100644 index 00000000..2fba5535 --- /dev/null +++ b/tests/syntax/fixtures_reference/whitespace_in_value.ftl @@ -0,0 +1,10 @@ +# Caution, lines 6 and 7 contain white-space-only lines +key = + first line + + + + + + + last line diff --git a/tests/syntax/fixtures_reference/whitespace_in_value.json b/tests/syntax/fixtures_reference/whitespace_in_value.json new file mode 100644 index 00000000..077e6867 --- /dev/null +++ b/tests/syntax/fixtures_reference/whitespace_in_value.json @@ -0,0 +1,26 @@ +{ + "type": "Resource", + "body": [ + { + "type": "Message", + "id": { + "type": "Identifier", + "name": "key" + }, + "value": { + "type": "Pattern", + "elements": [ + { + "type": "TextElement", + "value": "first line\n\n\n\n\n\n\nlast line" + } + ] + }, + "attributes": [], + "comment": { + "type": "Comment", + "content": "Caution, lines 6 and 7 contain white-space-only lines" + } + } + ] +} diff --git a/tests/syntax/test_reference.py b/tests/syntax/test_reference.py new file mode 100644 index 00000000..8af459df --- /dev/null +++ b/tests/syntax/test_reference.py @@ -0,0 +1,64 @@ +from __future__ import unicode_literals +from six import with_metaclass + +import os +import sys +import json +import codecs +import unittest + +sys.path.append('.') + +from fluent.syntax import parse, ast as ftl + + +def read_file(path): + with codecs.open(path, 'r', encoding='utf-8') as file: + text = file.read() + return text + + +fixtures = os.path.join( + os.path.dirname(__file__), 'fixtures_reference') + + +class TestReferenceMeta(type): + def __new__(mcs, name, bases, attrs): + + def gen_test(file_name): + def test(self): + ftl_path = os.path.join(fixtures, file_name + '.ftl') + ast_path = os.path.join(fixtures, file_name + '.json') + + source = read_file(ftl_path) + expected = read_file(ast_path) + + ast = parse(source) + for entry in ast.body: + if isinstance(entry, ftl.Junk): + entry.annotations = [] + + self.assertEqual( + ast.to_json(with_spans=False), json.loads(expected)) + + return test + + for f in os.listdir(fixtures): + file_name, ext = os.path.splitext(f) + + if ext != '.ftl': + continue + + # Skip fixtures which are known to differ between the reference + # parser and the tooling parser. + if file_name in ('leading_dots', 'variant_lists'): + continue + + test_name = 'test_{}'.format(file_name) + attrs[test_name] = gen_test(file_name) + + return type.__new__(mcs, name, bases, attrs) + + +class TestReference(with_metaclass(TestReferenceMeta, unittest.TestCase)): + maxDiff = None From cd69824e442fa0548275ccb58635b445182cea54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Fri, 7 Dec 2018 06:45:17 -0500 Subject: [PATCH 4/5] BaseNode.to_json() now accepts a processing function --- fluent/syntax/ast.py | 15 +++++++-------- tests/syntax/test_reference.py | 13 ++++++++----- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/fluent/syntax/ast.py b/fluent/syntax/ast.py index b7ee24bb..7ecd1826 100644 --- a/fluent/syntax/ast.py +++ b/fluent/syntax/ast.py @@ -3,13 +3,13 @@ import json -def to_json(value, **kwargs): +def to_json(value, fn=None): if isinstance(value, BaseNode): - return value.to_json(**kwargs) + return value.to_json(fn) if isinstance(value, list): - return list(to_json(item, **kwargs) for item in value) + return list(to_json(item, fn) for item in value) if isinstance(value, tuple): - return list(to_json(item, **kwargs) for item in value) + return list(to_json(item, fn) for item in value) else: return value @@ -119,16 +119,15 @@ def equals(self, other, ignored_fields=['span']): return True - def to_json(self, with_spans=True): + def to_json(self, fn=None): obj = { - name: to_json(value, with_spans=with_spans) + name: to_json(value, fn) for name, value in vars(self).items() - if with_spans or name != 'span' } obj.update( {'type': self.__class__.__name__} ) - return obj + return fn(obj) if fn else obj def __str__(self): return json.dumps(self.to_json()) diff --git a/tests/syntax/test_reference.py b/tests/syntax/test_reference.py index 8af459df..a849cd04 100644 --- a/tests/syntax/test_reference.py +++ b/tests/syntax/test_reference.py @@ -25,6 +25,13 @@ def read_file(path): class TestReferenceMeta(type): def __new__(mcs, name, bases, attrs): + def remove_untested(obj): + if obj['type'] == 'Junk': + obj['annotations'] = [] + if 'span' in obj: + del obj['span'] + return obj + def gen_test(file_name): def test(self): ftl_path = os.path.join(fixtures, file_name + '.ftl') @@ -34,12 +41,8 @@ def test(self): expected = read_file(ast_path) ast = parse(source) - for entry in ast.body: - if isinstance(entry, ftl.Junk): - entry.annotations = [] - self.assertEqual( - ast.to_json(with_spans=False), json.loads(expected)) + ast.to_json(remove_untested), json.loads(expected)) return test From be596db0e6222d936022a8c0808c0e61f9003d06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sta=C5=9B=20Ma=C5=82olepszy?= Date: Fri, 7 Dec 2018 07:32:23 -0500 Subject: [PATCH 5/5] Use decode('unicode-escape') to unescape Unicode sequences --- fluent/syntax/parser.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/fluent/syntax/parser.py b/fluent/syntax/parser.py index a904aaf7..0fee34c2 100644 --- a/fluent/syntax/parser.py +++ b/fluent/syntax/parser.py @@ -4,12 +4,6 @@ from .stream import EOF, EOL, FluentParserStream from .errors import ParseError -try: - from __builtin__ import unichr as chr -except ImportError: - pass - - def with_span(fn): def decorated(self, ps, *args, **kwargs): @@ -593,8 +587,11 @@ def get_unicode_escape_sequence(self, ps, u, digits): codepoint = int(sequence, 16) if codepoint <= 0xD7FF or 0xE000 <= codepoint: - # It's a Unicode scalar value. - unescaped = chr(codepoint) + # It's a Unicode scalar value. The escape sequence is 4 or 6 digits + # long. Convert it to a 8-digit-long \UHHHHHHHH sequence and encode + # it as bytes, because in Python 3 decode is not available on str. + byte_sequence = "\\U{:08x}".format(codepoint).encode('utf-8') + unescaped = byte_sequence.decode('unicode-escape') else: # Escape sequences reresenting surrogate code points are # well-formed but invalid in Fluent. Replace them with U+FFFD