October 09, 2026

Event recogntion grammar for maze robot

 _

BNF grammars are known for programming language specification but are also useful for language games between speaker and hearer. A pointing game or an instruction following game can be realized with such a grammar. Reversing a pointing game results into an action recognition game in which a human controls a robot with arrow keys, and the grammar recognizes the demonstrated actions. Such an action recognition game was implemented in python as a simple maze game. The robot can move around and pick&place objects. 
The game doesn't need a large language model and not a vision language action model realized as a neural network, but its a symbolic approach based on a grammar.[1][2]

[1] Dantam, Neil, Irfan Essa, and Mike Stilman. "Linguistic transfer of human assembly tasks to robots." 2012 IEEE/RSJ International Conference on Intelligent Robots and Systems. IEEE, 2012.
[2] Wegner, Peter, and Dina Goldin. "Computation beyond Turing machines." Communications of the ACM 46.4 (2003): 100-102.
 

import re
import pygame


# -------------------- Configuration --------------------

COLS, ROWS = 20, 14
CELL = 36
GRID_W, GRID_H = COLS * CELL, ROWS * CELL
LOG_H = 180
WIDTH, HEIGHT = GRID_W, GRID_H + LOG_H

# '#' is a wall; '.' is a walkable cell.
MAZE = [
    "####################",
    "#....#.............#",
    "#....#..####..###..#",
    "#.......#.....#....#",
    "###.##..#..##.#.##..#",
    "#...##.....##......#",
    "#.######.####.###..#",
    "#........#.........#",
    "#.###.####.###.##..#",
    "#.....#......#.....#",
    "###.#.#.####.#.###..#",
    "#...#..........#...#",
    "#........#.........#",
    "####################",
]

# Named locations are recognized in the event log.
LOCATIONS = {
    "start": (1, 1),
    "junction": (7, 3),
    "workshop": (15, 5),
    "storage": (4, 9),
    "exit": (18, 12),
}

# Objects can be picked up and placed with G.
OBJECTS = {
    "red_key": (3, 3),
    "blue_box": (16, 9),
    "coin": (9, 11),
}

# Robot begins at the named "start" location.
robot = list(LOCATIONS["start"])
carried_object = None
event_log = ["Arrow keys: move    G: pick up / place"]

# Draw colors
WHITE = (255, 255, 255)
BLACK = (25, 25, 25)
WALL_COLOR = (70, 75, 85)
GRID_COLOR = (205, 205, 205)
ROBOT_COLOR = (30, 105, 220)
LOCATION_COLOR = (30, 135, 65)
OBJECT_COLORS = {
    "red_key": (205, 45, 45),
    "blue_box": (45, 80, 205),
    "coin": (190, 145, 0),
}


# -------------------- Event grammar --------------------

# Events emitted by the controls:
#
# <event>       ::= <move-event> | <grasp-event> | <ungrasp-event>
# <move-event>  ::= "movto" <location>
# <grasp-event> ::= "grasp" <object>
# <ungrasp-event> ::= "ungrasp" <object>
#
# <location> includes the named locations above and all walkable maze cells,
# represented as cell_X_Y. <object> includes the names in OBJECTS.
#
# The parser below checks both the event's form and its vocabulary.

def location_names():
    names = set(LOCATIONS)
    for y in range(ROWS):
        for x in range(COLS):
            if MAZE[y][x] != "#":
                names.add(f"cell_{x}_{y}")
    return names


LOCATION_NAMES = location_names()
OBJECT_NAMES = set(OBJECTS)

EVENT_PATTERNS = [
    ("movto", re.compile(r"^movto\s+([a-z][a-z0-9_]*)$")),
    ("grasp", re.compile(r"^grasp\s+([a-z][a-z0-9_]*)$")),
    ("ungrasp", re.compile(r"^ungrasp\s+([a-z][a-z0-9_]*)$")),
]


def parse_event(text):
    """Return (verb, argument) if a generated event matches the grammar."""
    for verb, pattern in EVENT_PATTERNS:
        match = pattern.fullmatch(text)
        if not match:
            continue

        argument = match.group(1)
        valid_names = LOCATION_NAMES if verb == "movto" else OBJECT_NAMES
        if argument in valid_names:
            return verb, argument

    return None


def log_recognized_event(text):
    """Only add events that are accepted by the grammar."""
    if parse_event(text) is not None:
        event_log.append(f"Recognized: {text}")
        del event_log[:-7]


def location_at(x, y):
    """Prefer a named location; otherwise use the cell's generated name."""
    for name, position in LOCATIONS.items():
        if position == (x, y):
            return name
    return f"cell_{x}_{y}"


# -------------------- Robot actions --------------------

def is_walkable(x, y):
    return (
        0 <= x < COLS
        and 0 <= y < ROWS
        and MAZE[y][x] != "#"
    )


def move_robot(dx, dy):
    x, y = robot
    nx, ny = x + dx, y + dy

    if not is_walkable(nx, ny):
        return

    robot[:] = [nx, ny]
    log_recognized_event(f"movto {location_at(nx, ny)}")


def toggle_grasp():
    global carried_object

    current_cell = tuple(robot)

    if carried_object is not None:
        # Place the held object on the robot's current cell.
        if current_cell in OBJECTS.values():
            event_log.append("Cannot place: this cell already has an object.")
            del event_log[:-7]
            return

        object_name = carried_object
        OBJECTS[object_name] = current_cell
        carried_object = None
        log_recognized_event(f"ungrasp {object_name}")
        return

    # Pick up an object on the robot's current cell.
    for object_name, position in OBJECTS.items():
        if position == current_cell:
            carried_object = object_name
            del OBJECTS[object_name]
            log_recognized_event(f"grasp {object_name}")
            return

    event_log.append("No object here to pick up.")
    del event_log[:-7]


# -------------------- Pygame setup and drawing --------------------

pygame.init()
screen = pygame.display.set_mode((WIDTH, HEIGHT))
pygame.display.set_caption("Robot Maze — BNF Event Log")
clock = pygame.time.Clock()

font = pygame.font.Font(None, 24)
small_font = pygame.font.Font(None, 19)
robot_font = pygame.font.Font(None, 28)


def draw():
    screen.fill(WHITE)

    # Maze
    for y in range(ROWS):
        for x in range(COLS):
            rect = pygame.Rect(x * CELL, y * CELL, CELL, CELL)
            if MAZE[y][x] == "#":
                pygame.draw.rect(screen, WALL_COLOR, rect)
            pygame.draw.rect(screen, GRID_COLOR, rect, 1)

    # Named locations
    for name, (x, y) in LOCATIONS.items():
        label = small_font.render(name, True, LOCATION_COLOR)
        screen.blit(label, (x * CELL + 3, y * CELL + 2))

    # Objects
    for name, (x, y) in OBJECTS.items():
        cx = x * CELL + CELL // 2
        cy = y * CELL + CELL // 2 + 4
        pygame.draw.circle(screen, OBJECT_COLORS[name], (cx, cy), 10)
        label = small_font.render(name, True, BLACK)
        screen.blit(label, (x * CELL + 2, y * CELL + CELL - 17))

    # Robot, displayed as R
    rx, ry = robot
    robot_label = robot_font.render("R", True, ROBOT_COLOR)
    robot_rect = robot_label.get_rect(
        center=(rx * CELL + CELL // 2, ry * CELL + CELL // 2)
    )
    screen.blit(robot_label, robot_rect)

    # White event log panel
    log_top = GRID_H
    pygame.draw.rect(screen, WHITE, (0, log_top, WIDTH, LOG_H))
    pygame.draw.line(screen, BLACK, (0, log_top), (WIDTH, log_top), 2)

    title = font.render("Recognized events", True, BLACK)
    screen.blit(title, (10, log_top + 8))

    if carried_object:
        held_text = small_font.render(f"Holding: {carried_object}", True, BLACK)
        screen.blit(held_text, (WIDTH - 150, log_top + 11))

    visible_events = event_log[-6:]
    for i, message in enumerate(visible_events):
        text = small_font.render(message, True, BLACK)
        screen.blit(text, (10, log_top + 38 + i * 21))

    pygame.display.flip()


# -------------------- Main loop --------------------

running = True

while running:
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            running = False

        elif event.type == pygame.KEYDOWN:
            if event.key == pygame.K_ESCAPE:
                running = False
            elif event.key == pygame.K_UP:
                move_robot(0, -1)
            elif event.key == pygame.K_DOWN:
                move_robot(0, 1)
            elif event.key == pygame.K_LEFT:
                move_robot(-1, 0)
            elif event.key == pygame.K_RIGHT:
                move_robot(1, 0)
            elif event.key == pygame.K_g:
                toggle_grasp()

    draw()
    clock.tick(30)

pygame.quit()
 

No comments:

Post a Comment