Müllsuch-Roboter mit neural encoding

In einer 2d Karte ist ein Roboter mit einem vision cone unterwegs, der Müllobjekte kartieren soll. Es gibt einen manuellen Modus wo der User den Roboter mittels Pfeiltasten steuert und einen KI Modus der mit [k] aktiviert wird.

Der Roboter gibt die Statusinformationen einmal als Textnachrichten auf dem Bildschirm aus und zusätzlich noch als neural encoding, dieser ist ein feature vector mit 10 numerischen Werten. Im KI Modus wird dieser Feature vector verwendet um die Aktionen des Roboter zu planen.

 import sys
import math
import random
import numpy as np
import pygame

# --- KONFIGURATION & FARBEN ---
GRID_WIDTH = 30
GRID_HEIGHT = 20
CELL_SIZE = 30

GRID_PIXEL_W = GRID_WIDTH * CELL_SIZE
GRID_PIXEL_H = GRID_HEIGHT * CELL_SIZE
UI_HEIGHT = 180
WINDOW_WIDTH = GRID_PIXEL_W
WINDOW_HEIGHT = GRID_PIXEL_H + UI_HEIGHT

# Farben (RGB)
COLOR_BG = (30, 30, 35)
COLOR_WALL = (50, 50, 60)
COLOR_TRASH = (210, 45, 45)
COLOR_UNKNOWN = (100, 100, 110)
COLOR_MAPPED_FREE = (240, 240, 245)
COLOR_MAPPED_TRASH = (255, 140, 0)
COLOR_ROBOT = (30, 144, 255)
COLOR_VISION = (255, 255, 0, 60)

# Neue UI-Farben (Weißer Hintergrund, schwarzer Text)
COLOR_UI_BG = (255, 255, 255)
COLOR_UI_TEXT = (0, 0, 0)

# Zell-Typen
EMPTY = 0
WALL = 1
TRASH = 2


class Robot:

    def __init__(self, x, y, grid_w, grid_h):
        self.x = x
        self.y = y
        self.grid_w = grid_w
        self.grid_h = grid_h
        self.orientation = 0  # 0=Rechts, 90=Unten, 180=Links, 270=Oben
        self.fov_angle = 90
        self.fov_range = 5

    def move(self, dx, dy, grid):
        new_x = self.x + dx
        new_y = self.y + dy

        if dx == 1:
            self.orientation = 0
        elif dx == -1:
            self.orientation = 180
        elif dy == 1:
            self.orientation = 90
        elif dy == -1:
            self.orientation = 270

        if (
            0 <= new_x < self.grid_w
            and 0 <= new_y < self.grid_h
            and grid[new_y][new_x] != WALL
        ):
            self.x = new_x
            self.y = new_y
            return True
        return False


class Environment:

    def __init__(self, w, h):
        self.w = w
        self.h = h
        self.grid = np.zeros((h, w), dtype=int)
        self.generate_map()

    def generate_map(self):
        self.grid[0, :] = WALL
        self.grid[-1, :] = WALL
        self.grid[:, 0] = WALL
        self.grid[:, -1] = WALL

        self.grid[4:14, 8] = WALL
        self.grid[6, 8:18] = WALL
        self.grid[12:18, 20] = WALL

        trash_positions = [
            (3, 3),
            (5, 12),
            (10, 15),
            (15, 4),
            (22, 8),
            (25, 15),
            (12, 2),
            (18, 16),
        ]
        for tx, ty in trash_positions:
            self.grid[ty, tx] = TRASH


class RobotSystem:

    def __init__(self):
        pygame.init()
        pygame.display.set_caption(
            "2D Robot Mapping & Neural Encoding Simulation"
        )
        self.screen = pygame.display.set_mode((WINDOW_WIDTH, WINDOW_HEIGHT))
        self.clock = pygame.time.Clock()
        self.font = pygame.font.SysFont("Arial", 16, bold=False)

        self.env = Environment(GRID_WIDTH, GRID_HEIGHT)
        self.robot = Robot(2, 2, GRID_WIDTH, GRID_HEIGHT)

        self.robot_map = np.full((GRID_HEIGHT, GRID_WIDTH), -1, dtype=int)
        self.status_message = "SYS_INIT: ROBOT ONLINE."
        self.feature_vector = np.zeros(10)

        # KI Steuerung
        self.ai_mode = False
        self.ai_move_timer = 0

    def compute_vision_and_map(self):
        visible_cells = set()
        trash_in_fov = []

        start_angle = self.robot.orientation - (self.robot.fov_angle / 2)
        end_angle = self.robot.orientation + (self.robot.fov_angle / 2)

        for angle_deg in np.linspace(start_angle, end_angle, num=30):
            rad = math.radians(angle_deg)
            dx = math.cos(rad)
            dy = math.sin(rad)

            for step in range(1, self.robot.fov_range + 1):
                cx = int(round(self.robot.x + dx * step))
                cy = int(round(self.robot.y + dy * step))

                if 0 <= cx < GRID_WIDTH and 0 <= cy < GRID_HEIGHT:
                    visible_cells.add((cx, cy))
                    cell_val = self.env.grid[cy][cx]
                    self.robot_map[cy][cx] = cell_val

                    if cell_val == TRASH:
                        dist = math.hypot(cx - self.robot.x, cy - self.robot.y)
                        trash_in_fov.append((cx, cy, dist))

                    if cell_val == WALL:
                        break
                else:
                    break

        self.robot_map[self.robot.y][self.robot.x] = EMPTY
        visible_cells.add((self.robot.x, self.robot.y))
        return visible_cells, trash_in_fov

    def update_neural_encoding(self, trash_in_fov):
        """Erzeugt den 10-dimensionalen semantischen Feature-Vektor."""
        v = np.zeros(10)

        # v0, v1: Position (normiert)
        v[0] = round(self.robot.x / (GRID_WIDTH - 1), 2)
        v[1] = round(self.robot.y / (GRID_HEIGHT - 1), 2)

        # v2-v5: Hindernis-Nähe (Vorne, Rechts, Hinten, Links) -> 1.0 = Wand nah
        dirs = [
            self.robot.orientation,
            (self.robot.orientation + 90) % 360,
            (self.robot.orientation + 180) % 360,
            (self.robot.orientation + 270) % 360,
        ]

        for i, d in enumerate(dirs):
            rad = math.radians(d)
            dx, dy = int(round(math.cos(rad))), int(round(math.sin(rad)))
            dist = 0
            cx, cy = self.robot.x, self.robot.y
            while True:
                cx += dx
                cy += dy
                dist += 1
                if (
                    not (0 <= cx < GRID_WIDTH and 0 <= cy < GRID_HEIGHT)
                    or self.env.grid[cy][cx] == WALL
                ):
                    break
            v[2 + i] = round(1.0 / max(dist, 1), 2)

        # v6: Müll im Sichtfeld (0.0 oder 1.0)
        v[6] = 1.0 if len(trash_in_fov) > 0 else 0.0

        # v7: Distanz/Nähe zum Müll (1.0 = sehr nah, 0.0 = weit weg/keiner)
        if len(trash_in_fov) > 0:
            min_dist = min([t[2] for t in trash_in_fov])
            v[7] = round(max(0.0, 1.0 - (min_dist / self.robot.fov_range)), 2)
        else:
            v[7] = 0.0

        # v8: Kartierungsfortschritt
        mapped_count = np.sum(self.robot_map != -1)
        v[8] = round(mapped_count / (GRID_WIDTH * GRID_HEIGHT), 2)

        # v9: Müll-Dichte / Erfassungsquote
        found_trash = np.sum(self.robot_map == TRASH)
        v[9] = round(min(1.0, found_trash / max(1, mapped_count * 0.1)), 2)

        self.feature_vector = v

    def ai_decide_move(self):
        """KI-Entscheidung: Greift AUSSCHLIESSLICH auf den Feature-Vektor zu."""
        vec = self.feature_vector

        # Mögliche Bewegungsrichtungen: [(dx, dy), Orientierung]
        moves = [
            ((1, 0), 0),  # Rechts
            ((0, 1), 90),  # Unten
            ((-1, 0), 180),  # Links
            ((0, -1), 270),  # Oben
        ]

        valid_moves = []
        for (dx, dy), orient in moves:
            # Zuordnung der Vektor-Wände (v2..v5) zur relativen Ausrichtung
            rel_angle = (orient - self.robot.orientation) % 360
            idx = int(rel_angle // 90) + 2

            # Wenn die Wand nicht direkt davor steht (v < 1.0 ist frei)
            if vec[idx] < 1.0:
                valid_moves.append(((dx, dy), orient, vec[idx]))

        if not valid_moves:
            return

        # STRATEGIE 1: Müll jagen (wenn v6 == 1.0)
        if vec[6] == 1.0:
            best_move = None
            best_score = -999

            for (dx, dy), orient, wall_proximity in valid_moves:
                # Teste virtuell, wie sich v7 verändern würde
                # Ausrichtungsauswertung in Richtung Müll
                score = -wall_proximity * 2.0
                if orient == self.robot.orientation:
                    score += 2.0  # Bevorzuge Vorwärtsbewegung zum Müll

                if score > best_score:
                    best_score = score
                    best_move = (dx, dy)

            if best_move:
                self.robot.move(best_move[0], best_move[1], self.env.grid)
                return

        # STRATEGIE 2: Karte erkunden (Wänden ausweichen, geradeaus bevorzugen)
        best_move = None
        best_score = -999

        for (dx, dy), orient, wall_proximity in valid_moves:
            # Score: Viel Platz (geringer Wall-Proximity Wert) + Kontinuität
            score = (1.0 - wall_proximity) * 3.0
            if orient == self.robot.orientation:
                score += 1.5  # Vorwärtsdrang
            score += random.uniform(0.0, 0.5)  # Zufallskomponente gegen Schleifen

            if score > best_score:
                best_score = score
                best_move = (dx, dy)

        if best_move:
            self.robot.move(best_move[0], best_move[1], self.env.grid)

    def update_telemetry(self, trash_in_fov):
        mode_str = "AI-MODE (AUTONOMOUS)" if self.ai_mode else "MANUAL (KEYBOARD)"
        msg_parts = [f"MODE: {mode_str}", f"POS:[{self.robot.x},{self.robot.y}]"]

        if trash_in_fov:
            closest = min(trash_in_fov, key=lambda item: item[2])
            msg_parts.append(
                f"ALERT: TRASH DETECTED AT [{closest[0]},{closest[1]}]"
            )
        else:
            msg_parts.append("SCANNING... NO TRASH IN FOV")

        total_trash_mapped = np.sum(self.robot_map == TRASH)
        msg_parts.append(f"MAPPED TRASH TOTAL: {total_trash_mapped}")

        self.status_message = " | ".join(msg_parts)

    def draw(self, visible_cells):
        self.screen.fill(COLOR_BG)

        # 1. Gitter & Map zeichnen
        for y in range(GRID_HEIGHT):
            for x in range(GRID_WIDTH):
                rect = pygame.Rect(
                    x * CELL_SIZE, y * CELL_SIZE, CELL_SIZE, CELL_SIZE
                )
                mapped_val = self.robot_map[y][x]
                real_val = self.env.grid[y][x]

                if mapped_val == -1:
                    color = COLOR_UNKNOWN
                elif mapped_val == WALL:
                    color = COLOR_WALL
                elif mapped_val == TRASH:
                    color = COLOR_MAPPED_TRASH
                else:
                    color = COLOR_MAPPED_FREE

                pygame.draw.rect(self.screen, color, rect)

                if (x, y) in visible_cells and real_val == TRASH:
                    pygame.draw.circle(
                        self.screen, COLOR_TRASH, rect.center, CELL_SIZE // 3
                    )

                pygame.draw.rect(self.screen, (50, 50, 50), rect, 1)

        # 2. Vision Cone
        rx_pix = self.robot.x * CELL_SIZE + CELL_SIZE // 2
        ry_pix = self.robot.y * CELL_SIZE + CELL_SIZE // 2

        cone_surface = pygame.Surface(
            (GRID_PIXEL_W, GRID_PIXEL_H), pygame.SRCALPHA
        )
        for cx, cy in visible_cells:
            c_rect = pygame.Rect(
                cx * CELL_SIZE, cy * CELL_SIZE, CELL_SIZE, CELL_SIZE
            )
            pygame.draw.rect(cone_surface, COLOR_VISION, c_rect)
        self.screen.blit(cone_surface, (0, 0))

        # 3. Roboter
        pygame.draw.circle(
            self.screen, COLOR_ROBOT, (rx_pix, ry_pix), CELL_SIZE // 2 - 2
        )
        rad = math.radians(self.robot.orientation)
        end_x = rx_pix + math.cos(rad) * (CELL_SIZE // 2)
        end_y = ry_pix + math.sin(rad) * (CELL_SIZE // 2)
        pygame.draw.line(
            self.screen, (255, 255, 255), (rx_pix, ry_pix), (end_x, end_y), 3
        )

        # 4. UI / Textfeld (WEISSER HINTERGRUND & SCHWARZER TEXT)
        ui_rect = pygame.Rect(0, GRID_PIXEL_H, WINDOW_WIDTH, UI_HEIGHT)
        pygame.draw.rect(self.screen, COLOR_UI_BG, ui_rect)
        pygame.draw.line(
            self.screen,
            (200, 200, 200),
            (0, GRID_PIXEL_H),
            (WINDOW_WIDTH, GRID_PIXEL_H),
            2,
        )

        # Statuszeile
        txt_surface = self.font.render(
            f"> STATUS: {self.status_message}", True, COLOR_UI_TEXT
        )
        self.screen.blit(txt_surface, (15, GRID_PIXEL_H + 15))

        # Neural Vector Zeile
        vec_title = self.font.render(
            "> NEURAL FEATURE VECTOR [v0..v9]:", True, COLOR_UI_TEXT
        )
        self.screen.blit(vec_title, (15, GRID_PIXEL_H + 45))

        vec_str = np.array2string(
            self.feature_vector, precision=2, suppress_small=True
        )
        vec_surface = self.font.render(
            f"  {vec_str}", True, COLOR_UI_TEXT
        )
        self.screen.blit(vec_surface, (15, GRID_PIXEL_H + 65))

        # Vektor-Legende
        legend = "[PosX, PosY, DistN, DistE, DistS, DistW, TrashVisible, TrashDist, MapProgress, TrashDensity]"
        leg_surface = self.font.render(f"  {legend}", True, COLOR_UI_TEXT)
        self.screen.blit(leg_surface, (15, GRID_PIXEL_H + 90))

        # Steuerungshinweis
        ctrl_str = "Steuerung: Pfeiltasten (Manuell) | Taste 'K' drücken (KI-Modus Umschalten)"
        ctrl_surface = self.font.render(ctrl_str, True, COLOR_UI_TEXT)
        self.screen.blit(ctrl_surface, (15, GRID_PIXEL_H + 130))

        pygame.display.flip()

    def run(self):
        running = True
        while running:
            self.clock.tick(30)
            self.ai_move_timer += 1

            for event in pygame.event.get():
                if event.type == pygame.QUIT:
                    running = False
                elif event.type == pygame.KEYDOWN:
                    if event.key == pygame.K_k:
                        # KI-Modus umschalten
                        self.ai_mode = not self.ai_mode
                    elif not self.ai_mode:
                        # Manuelle Steuerung nur, wenn KI aus ist
                        if event.key == pygame.K_UP:
                            self.robot.move(0, -1, self.env.grid)
                        elif event.key == pygame.K_DOWN:
                            self.robot.move(0, 1, self.env.grid)
                        elif event.key == pygame.K_LEFT:
                            self.robot.move(-1, 0, self.env.grid)
                        elif event.key == pygame.K_RIGHT:
                            self.robot.move(1, 0, self.env.grid)

            # KI-Schritt ausführen (alle 5 Frames für eine flüssige Bewegung)
            if self.ai_mode and self.ai_move_timer >= 5:
                self.ai_decide_move()
                self.ai_move_timer = 0

            # Sensorik & System-Updates
            visible_cells, trash_in_fov = self.compute_vision_and_map()
            self.update_neural_encoding(trash_in_fov)
            self.update_telemetry(trash_in_fov)

            # Zeichnen
            self.draw(visible_cells)

        pygame.quit()
        sys.exit()


if __name__ == "__main__":
    sim = RobotSystem()
    sim.run()

Car racing with grounded language

 The screenshot shows a game AI prototype based on grounded language. The current game state is converted first into textual description which is then processed by the rule system to determine the next action. The textual log file at the bottom of the screen is the decision making logic.

The main advantage of grounded language is, that it compress the game state into a small amount of possible states. Instead of describing the racing game as a vision task which contains of a 2d pixel game, the perception system converts the game into a list of words which are "lane is safe, lane is clear, collision". This small amount of discrete states makes it easier to program a rule system which determines what the car is doing next in the game.

From a programming perspective its a mix of racing video game on top of the screen and text adventure at the bottom. Even it is possible to implement the game AI on outdated 8bit homecomputers from the 1980s, the software is very new because it assumes a certain understanding of Artificial intelligence. AI isn't described as algorithms but AI shown in the prototype is a textual description of the game state.

Playing a videogame with features

 The pictures shows a neural network which determines the next action for a character in a videogame. The network needs 20 input features which are player_pos, energy, last jump height, distance to enemy and some other features. The hidden layer of the network determines the current situation its a feature-to-information layer. The output layer provides the action itself submitted to the game pad.

The intelligence of the neural network doesn't depend on the training algorithm because its a simple backpropagation algorithm with nu tuning, but the intelligence depends on the input features. These features were chosen manual by a human to mirror the current game state in a meaningful way. The goal is to provide the smallest amount of features and ensure that the data are updated in realtime. 

Head up display for a grasping robot with keyframes

Information overload with head up display

 

Robot control with head up displays

In contrast to a famous assumption, modern robotics isn't working with algorithms or neural networks but the basic building block is graphical user interface, namely a head up display (HuD). The HuD solves the symbol grounding problem. Typical elements are: bounding boxes around detected objects, text labels for describing the content of a bounding box, another text box for showing the inner voice of rhe robot.

These ingredients are enough to program an advanced artificial intelligence which can solve complex problems. The HuD including the mentioned bounding boxes acts as a communication layer. It ensures that the computer understands basic commands like "move to shelf and grasp the box". A certain high level command is converted into a visual pictures in the HuD, e.g. the word "shelf" is referencing to a bounding box with the label "shelf" which has a 2d position on the screen.

Programming a Head up display for an existing video game is a demanding task but can be solved with standard programming techniques. Most videogames created since the 1980s have a built in debug mode which comes close to a head up display. In the debut mode, all the sprites on the screen are highlighted with frames and sometimes the name of the objects are shown as textual overlay. The combination of graphical display plus textual overlay is the main principle of a head up display and also the main principle of grounded language. So the HuD itself acts as technology for enabling artificial intelligence.

Let me give another example to demonstrate the advantages: Suppose the head up display for a warehouse robot videogame was activated. The user sees some bounding boxes on the screen for highlighting objects in the map like charging station, corridor, shelf A, shelf B, green box, red box. Also the inner voice of the robot is shown a text frame and contains:

"I'm standing at position (3,2). My battery level is 80%, my goal is to fetch the red box from shelf A, the planned trajectory is shown as arrows in the map".

So the initial situation for the robot is, that an annotated HuD is visible which labels objects and mentions the current goal. These information can be translated into actions for the robot. All what the AI of the robot has to do is to compile these information and decide what to do next. From an AI perspective its an instruction following task with an aciivated head up display.

A head up display provides a cognitive space. The shown bouding boxes and labels are creating a symbolic representation of the world. The world of the robot can be described in terms from the head up display. Its no longer a mathematical space and not a 3d space but the reality introduced by the HuD consists of words, locations of items and goals from the inner voice. Such a high level space can be processed by a computer because the amount of possible states is small. There are not millions of possible objects but the HuD shows only 6 different objects in a map. and the inner voice doesn't display millions of possible actions, but the inner voice describes clearly what the current situation is, and what the desired goal state is, similar to a text adventure.

Head up display with inner voice for a kitchen robot in pixelart

Short history of ingame AI

 Apart from automation tasks in a factory, there are major attempts available since the 1980s to build intelligent ingame characters targetted towards videogames. This subject seems to be easier to solve because in a videogame all the information are known.

Typical ingame AI in the 1980s was realized with Finite state machines. Especially the pacman game is using this single technique to control the ghosts. Another famous approach is depth first search used in board games likes chess and Nine men's morris.

Both concepts have major disadvantages. A finite state machine is difficult to program and a game state traversal in chess needs a lots of CPU ressources. Until around the year 2000 there were no improvements available. Even if finite state machine have evolved into behavior trees it was also hard to implement.

The main challenge in programming an ingame AI can be summarized as the reality gap between the videogame and the internal representation of the AI agent. A Finite state machine has a certain perspective towards the game encoded in state. For example a pacman ghost has states like attack, evade, idle, random and these states are applied to the current situation. In most cases the reality of a game is more complex than the game AI representation which causes an asynchronous situation. In other word, the game AI isn't communicating enought with the videogame and this explains its poor decision making.

To overcome the bottleneck of ingame AI created until the year 2000 the focus should be on the communication between a videogame and an ingame AI. For reason of simplication there is a virtual referee who is talking to the ingame AI in natural language. This virtual referee is the source of intelligence. He will guid the AI agent. In case of Pacman the referee might say to a ghost "move to upper left", in case of chess the referee might say "protect the center".

Such kind of textual interaction solves the former reality gap. The game AI gets a constant flow of commands from the referee and the only obstacle is to understand and execute them.

Lets compare old school ingame AI with modern communication based AI. The typical AI for a videogame before the year 2010 was realized as a software project. The idea was to encode the knowledge in the source code and make the AI smart by itself. The goal was that the AI acts independent from its environment and has all the needed knowledge and all the needed algorithm as internal software modules for pathfinding, decision making, perception and case based reasoning. Of course it was very complicated to program such an AI but there was no alternative available.

In contrast, modern AI created after the year 2010 is working with the extend mind thesis. The source of knowledge and intelligence is located ooutside of the game bot, either in the game engine, in a virtual referee or in a human operator. There is no need to encode knowledge into the AI itself but the AI is realized as parser for external commands, similar to a receiver in a RC Car teleoperation. The receiver listens to the signals and converts into action. this principle results into a minimalistic software which is much easier to realize and is more flexible at the same time.

The surprising situation is, that technically such a concept was realized in the 1980 already but it was recognized as a here to stay technology. In case of text adventure likes Zork and early role playing games, the human user was entering text commands which were executed by the game engine. So there was no AI available as a compuational engine, but there was only a parser available which executed a two word command.

Such a parser has no reality gap because it has no internal representation. The external human operator is responsible that the avatar is reaching its goal. The parser is only a command receiver.

Vision and language dataset generator

The screenshot consists of a random scene generator plus a textual annotation for a food collecting robot. The algorithm generates a maze including food items, and the text widget shows the description of the scene.

Such a setup is useful to generate a synthetic dataset with picture/text pairs to train a neural network.

Textuelle Interaktion für Lagerroboter

 Textuelle Interaktion für Lagerroboter

Das folgende game log zeigt die interaktion zwischen 3 Agenten: eine game engine, eine Spielerin und einen internen Lehrer. Die Interaktion erfolgt ausschließlich über ein Text terminal. Das Ziel ist das Text adventure zu gewinnen indem Kommandos eingegeben werden und der interne Lehrer befragt wird.

=== WAREHOUSE ROBOT SIMULATION ===
KI_Spielerin: "starte auf Position A1"
Game_engine: "Roboter bereit. Ziel: 10 Kisten zu Regal B5 transportieren."
KI_Spielerin: "scanne Regal B5"
Game_engine: "Regal B5: 3/10 Kisten fehlen."
Internal_Teacher: "Nutze Greifarm Modus 3 für Kisten >20kg."
KI_Spielerin: "greife Kiste bei A1"
Game_engine: "Kiste erfasst. Gewicht: 25kg."
KI_Spielerin: "bewege zu B5"
Game_engine: "Weg frei. Transport startet."
KI_Spielerin: "platzieren Kiste"
Game_engine: "Kiste in Regal B5 abgelegt. 1/10 erledigt."
KI_Spielerin: "scanne Regal B5"
Game_engine: "Regal B5: 2/10 Kisten fehlen."
Internal_Teacher: "Prüfe Batteriestand. Warnung: 15%."
KI_Spielerin: "fahre zur Ladestation"
Game_engine: "Ladestation erreicht. Ladevorgang startet."
KI_Spielerin: "lade bis 80%"
Game_engine: "Batterie: 80%. Transport fortsetzen."
KI_Spielerin: "bewege zu A1"
Game_engine: "Position A1 erreicht."
KI_Spielerin: "greife nächste Kiste"
Game_engine: "Kiste erfasst. Gewicht: 18kg."
KI_Spielerin: "bewege zu B5"
Game_engine: "Weg blockiert. Hindernis: Palette bei A3."
KI_Spielerin: "frage nach Lösung"
Internal_Teacher: "Umfahren oder Hindernis entfernen. Risiko: 2 Min Verzögerung."
KI_Spielerin: "umfahre Hindernis"
Game_engine: "Alternative Route berechnet. Transport startet."
KI_Spielerin: "platzieren Kiste"
Game_engine: "Mission 50% abgeschlossen."
=== ENDE ===

Playing a videogame with a textbox

 For arduino microcontrollers there is a standard display available with 20x4 characters. Such a small text display is a great choice for demonstrating grounded language in action. In an example jump'n'run sidesrolling game, the following text boxes were generated by the event detection engine:

Example 1
-----------
PLAYER ON GROUND
JUMP READY         
COIN +1   TOTAL 5  
ENEMY NEAR -COVER  

Example 2
-----------
DASH COOLDOWN 1.2s
PLATFORM AHEAD 3m 
SPIKE! STEP BACK   
HEALTH 4/5  POWERUP

Example 3
-----------
FELL -1LIFE       
RESPAWN AT CHECKPT 
TIME 02:14       
KEY ACQUIRED  DOOR

Example 4
-----------
SPEED BOOST ACTIVE
ENEMY HIT x2     
COMBO 3X  +50PTS 
SECRET PATH DETECTʼD

From a technical perspective such a textbox is highly efficient. The text occupies very few amount of RAM and because the repeating pattern it can be compressed further.

Even if the description is formulated on a high abstraction layer, its possible to use these information to play the game with an Artificial intelligence. All what is needed are a list of rules for determining what to do in each situation. These rules are not applied to the graphical videogame at 800x600 Pixel resolution but the rules are applied to the text box.

Game engines with grounded language

Most video games are programmed with a game engine. The game engine stores the objects on the screen and renders them to the monitor. Such a setup ensures that the game can be implemented in Python with the constraint that a human player interacts with the game engine.

To make sure, that the game can also be played with artificial intelligence and additional module is needed which is a textual buffer. The buffer converts the output of the game egine into textual sentences e.g. "player is left, red obstacle ahead". The assumption of the textual buffer is, that the game should be rendered to a mini text terminal which has only 16x2 characters. The challenge is that a complex game gets compressed into such a small text buffer. This is only possible with a vocabulary of words. Converting the normal videoscreen of 800x600 pixels into a list of words can be realized with a computer program in realtime.

Annotating a warehouse robot

The table shows a simple warehouse game played in a 1d corridor. The robot R has to reach the target T and charge its battery at the Start S. Implementing such a mini game is usually realized with an array for storing the position of the objects.

What makes the simulation more demanding is the introduction of grounded language in the form of [tags]. These tags are used to describe the situation on a semantic level. In a DIKW pyramid, the 1d ascii corridor might be the data level, while the tags are the information level. 

1D ASCII Corridor	Annotation with [Tags]
[S R . . . . . T]	[at_start] [near_station] [status_idle] [path_clear]
[. S . R . . . T]	[moving_east] [leaving_station] [battery_optimal]
[. S . . R . . T]	[moving_east] [mid_corridor] [calculating_distance]
[. S . . . R . T]	[near_target] [decelerating] [scanning_area]
[. S . . . . R T]	[at_target] [loading_process] [task_active]
[. S . . . . T R]	[target_passed] [reversing] [adjusting_position]
[. S . . . R . T]	[moving_west] [returning_to_base] [low_battery_warning]
[R S . . . . . T]	[docking] [at_station] [recharging] [task_complete]

Grounded text to action for playing Maniac Mansion

 In addition to the previous attemps to play the point&click adventure Maniac mansion with a large language model here is a more compressed repreentation. Its a 3 column table with a timecode, a textual desription, and low level mouse actions.

The textual description is located on the information layer of the DIWK pyramid, while the mouse movements are on the bottom data layer.

Timecode    Textual Description    ScummVM Mouse Movements / Interaction
00:05    Select Character: Bernard    Move cursor to Bernard's portrait (bottom right); Left-Click.
00:08    Move to Front Gate    Move cursor to far right of driveway; Left-Click.
00:15    Walk to Front Door    Move cursor to porch steps; Left-Click.
00:20    Action: "Pull" Door Mat    Click "Pull" (verb pane); Click "Door Mat" (on porch floor).
00:24    Action: "Get" House Key    Click "Get" (verb pane); Click "Entrance Key" (revealed on floor).
00:28    Action: "Use" Key on Door    Click "Use"; Click Key (inventory); Click "Front Door".
00:32    Enter Mansion (Main Hall)    Move cursor to open doorway; Left-Click.
00:40    Action: "Get" Flashlight    Walk to the small table near the stairs; Click "Get"; Click "Flashlight".

The main task for the sofrware is translation. A high level textual description gets converted into low level action. E.g.:
textual description= Action: Use Key on Door
mouse movement=Click "Use"; Click Key (inventory); Click "Front Door".

In other words the DIKW pyramid is mostly an abstraction mechanism which consists of different details for the same task. The AI for playing Maniac Mansion hasn't decide anything, but the AI takes a textual description as input and generates low level mouse movements as output.

Here is the workflow how to play the game with an AI. The human user has to provide the textual description what to do in each scene. For example the human enters "walk to front door". This input command is converted by the computer into mouse actions on the screen and executed by the computer. So the Maniac Mansion game gets teleoperated with an advanced textual interface. This interface reduces the workload for the human operator. He is no longer forced to move the mouse directly on the verbs and the objects, but the human enters text into the command line.

Its a bit complicated to explain why such a DIWK workflow works in reality. From a technical perspective, natural language was utilized as an abstraction mechanis to reduce complexity. Instead of solving the original task of moving the mouse on the screen and click on items, the new task to provide a textual walk through which gets converted automatically into mouse movemens.

This abstraction mechanism works only because natural language, here English, is a powerful tool. It provides all the needed vocabulary including grammar to formulate complex tasks. There is no need to develop computer algorithm, neural networks or cognitive architectures, but natural language itself is the asset for enabling artificial intelligence. 

Human to robot interaction with a dikw pyramid

Maniac Mansion is a well known point&click adventure. With the help of a walk through tutorial its possible to win the game. The standard tutorial consists of keypoints and full sentences written in English which can be read by humans but can't be executed by a computer. With a converter from high level to a low level layer its possible to transform the walk through tutorial into machine readable commands. The process is demonstrated with the following json code adressing the kitchen scene:

{
  "card_id": "MM_KITCHEN_01",
  "scene_title": "The Mansion Kitchen - ScummVM Navigation",
  "content": {
    "textual_description": {
      "objective": "Enter the kitchen to retrieve the Small Key from the counter while staying alert for Nurse Edna.",
      "key_points": [
        "The kitchen is located through the first door on the right in the main hallway.",
        "Crucial Item: The Small Key is sitting on the counter near the sink.",
        "Hazard: Opening the refrigerator triggers a cutscene/event that can lead to capture.",
        "Exit Strategy: Use the door to the far right to enter the Dining Room if the hallway is blocked."
      ]
    },
    "low_level_representation": {
      "engine_context": "ScummVM - 320x200 Resolution (Original Scale)",
      "mouse_interactions": [
        {
          "step": 1,
          "verb_action": "PICK UP",
          "verb_coordinates": { "x": 40, "y": 175 },
          "target_object": "Small Key",
          "target_coordinates": { "x": 165, "y": 115 },
          "result": "Key added to character inventory."
        },
        {
          "step": 2,
          "verb_action": "WALK TO",
          "verb_coordinates": { "x": 10, "y": 165 },
          "target_location": "Dining Room Door",
          "target_coordinates": { "x": 305, "y": 110 },
          "result": "Character transitions to the next room."
        }
      ],
      "safety_note": "Avoid clicking 'OPEN' (x: 10, y: 175) on the Refrigerator (x: 240, y: 90) unless you have a specific distraction planned."
    }
  }
}

Both layers (low level and high level) are describing the same scene which is to enter the kitchen and fetch the key. The difference is, that that the layers have a different abstraction level. The high level layer is prefered by humans and mirrors how humans are thinking and how they are using language. In contrast, the low level layer is prefered by machines who are programmed with a logic oriented mathematical notation.

The converter has the task to translate between these layer which is known as the symbol grounding problem. Solving the grounding problem means to improve human to machine interaction.

Solving the first scene in Maniac Mansion with a DIKW pyramid

 Symbol grounding means basically to convert abstract description into detailed description. A concrete example with 3 layers for the first scene of the point&click adventure Maniac Mansion is shown next. The textual description can be understood by a human easily but can't be submitted directly to a computer. In contrast, the low level "pyautogui_commands" are hard to read for a human but can be processed by a computer program with ease.

Symbol grounding means basically that an algorithm converts high level descripiton into low level commands. With such a grounding algorithm its possible to script the game by providing textual description and the Artificial intelligence converts these description into mouse movements which are submitted to the SCUMMVM engine.

{
  "notecard_id": 1,
  "scene_title": "The Front Yard",
  "content": {
    "textual_description": {
      "objective": "Gain entry to the Edison Mansion.",
      "key_points": [
        "Start with Dave outside the main gate.",
        "Walk toward the front door of the mansion.",
        "The door is locked; the key is hidden nearby.",
        "Look under the doormat to find the silver key.",
        "Use the key to unlock the door and enter."
      ]
    },
    "low_level_representation": {
      "resolution_reference": "800x600",
      "actions": [
        {
          "step": 1,
          "action": "Select Verb: WALK TO",
          "pixel_coords": [120, 480],
          "note": "Clicking the 'Walk to' verb in the UI tray."
        },
        {
          "step": 2,
          "action": "Target: Front Door",
          "pixel_coords": [400, 300],
          "note": "Moving the character to the mansion entrance."
        },
        {
          "step": 3,
          "action": "Select Verb: PULL",
          "pixel_coords": [250, 480],
          "note": "Preparing to move the mat."
        },
        {
          "step": 4,
          "action": "Target: Doormat",
          "pixel_coords": [400, 420],
          "note": "Revealing the hidden key."
        },
        {
          "step": 5,
          "action": "Select Verb: PICK UP",
          "pixel_coords": [50, 520],
          "note": "Collecting the key."
        }
      ]
    },
    "pyautogui_commands": [
      "import pyautogui",
      "pyautogui.PAUSE = 0.5",
      "# Walk to door",
      "pyautogui.click(120, 480)",
      "pyautogui.click(400, 300)",
      "# Pull mat",
      "pyautogui.click(250, 480)",
      "pyautogui.click(400, 420)",
      "# Pick up key",
      "pyautogui.click(50, 520)",
      "pyautogui.click(405, 425)"
    ]
  }
}

Abstieg in der DIKW Pyramide am Beispiel Zak mckracken

 Damit ein Large language model ein Videospiel automatisiert durchspielt braucht es mehrere Ebenen aus der DIKW Pyramide. Auf layer 3 (knowledge) wird eine Spielszene in Stichpunkten beschrieben auf einer sehr hohen Abstraktionsschicht. Dies wird dann in den layer2 übersetzt, der viel präziser ist aber weniger leicht zu lesen für einen Menschen und schlußendlich auf den Layer1 transformiert der die low level Daten Ebene darstellt. Der Layer1 kann dann an die Game engine gesendet werden, also an die ScummVM welche das point&click adventure ausführt.

Hier alle 3 layer der DIKW pyramide in einer übersichtlichen json notation.

{
  "game": "Zak McKracken and the Alien Mindbenders",
  "card_id": 1,
  "title": "Morgenroutine in San Francisco",

  "representation_1_natural_language": {
    "format": "Karteikarte (Menschlich)",
    "content": [
      "Wache in Zaks Schlafzimmer auf.",
      "Nimm das Aquarium-Netz unter dem Bett.",
      "Gehe ins Wohnzimmer und nimm die Fernbedienung vom Fernseher.",
      "Gehe in die Küche.",
      "Nimm das stumpfe Brotmesser aus der Spüle.",
      "Öffne den Kühlschrank und nimm das Ei."
    ]
  },

  "representation_2_intermediate_logic": {
    "format": "Text-to-Action Reasoning (Zwischenschritt)",
    "note": "Hier werden implizite Aktionen und Raumwechsel für die KI logisch explizit gemacht.",
    "logic_chain": [
      {"state": "Room: Bedroom", "goal": "Inventory: Fishnet", "sub_action": "PickUp(Fishnet, under_bed)"},
      {"state": "Room: Bedroom", "goal": "Change Room", "sub_action": "WalkTo(Door_West)"},
      {"state": "Room: Living Room", "goal": "Inventory: Remote", "sub_action": "PickUp(Remote_Control, on_TV)"},
      {"state": "Room: Living Room", "goal": "Change Room", "sub_action": "WalkTo(Door_North)"},
      {"state": "Room: Kitchen", "goal": "Inventory: Knife", "sub_action": "PickUp(Bread_Knife, in_Sink)"},
      {"state": "Room: Kitchen", "goal": "Access Fridge", "sub_action": "Open(Refrigerator)"},
      {"state": "Room: Kitchen", "goal": "Inventory: Egg", "sub_action": "PickUp(Egg, inside_Fridge)"}
    ]
  },

  "representation_3_low_level_scumm": {
    "format": "SCUMM Engine Executable (Low Level)",
    "note": "Direkte Opcode-artige Anweisungen, die Objekten IDs und Verben zuordnen (fiktive IDs).",
    "commands": [
      {"op": "CUTSCENE_START"},
      {"op": "PICK_UP", "obj_id": 142, "comment": "Fishnet"},
      {"op": "WALK_TO_OBJECT", "obj_id": 201, "comment": "Door to Living Room"},
      {"op": "PICK_UP", "obj_id": 155, "comment": "Remote Control"},
      {"op": "WALK_TO_OBJECT", "obj_id": 202, "comment": "Door to Kitchen"},
      {"op": "PICK_UP", "obj_id": 160, "comment": "Bread Knife"},
      {"op": "OPEN", "obj_id": 175, "comment": "Refrigerator"},
      {"op": "PICK_UP", "obj_id": 176, "comment": "Egg"},
      {"op": "CUTSCENE_END"}
    ]
  }
}

 

Das interessante an dem Ansatz ist die Abwesenheit einer künstlichen Intelligenz im klassischen Sinne. Es gibt also kein neuronales Netz oder einen Reinforcement Learning algorithmus welches das Spiel durchspielt sondern die KI wurde so implementiert, dass sie zwischen den layern der DIWK pyramide eine Übersetzung ausführt. Wenn man möchte dass im Videospiel eine andere Aktion ausgeführt wird, schreibt man in den obersten Layer andere Stichworte hinein, die werden dann in echtzeit in die unteren Layer transformiert und ausgeführt. Das prinzip ist vergleichbar mit Unix pipes, also einer Filterkaskade die input in output übersetzt. 

LLMs als ingame AI

Large language modelle genießen eine große Popularität doch bisher wird dessen Potential nur teilweise ausgeschöpft. So finden LLM ihren Einsatz als Suchmaschine, zum Programmieren oder um Fragen zu beantworten. Hingegen werden sie kaum bis gar nicht eingesetzt um Computerspiele automatisiert durchzuspielen. Zwar gibt es neuere Entwicklungen wie Deepmind Sima 2, was durchaus in der Lage ist innerhalb von virtuellen Welten zu agieren, doch ist der Tokenverbrauch hoch und es gibt nur wenige Dokumentationen dazu.

Heutige LLM sind durchaus imstande in virtuellen Welten zu agieren. Als Beispiel sei hier eine stichpunktartige Komplettlösung für das Maniac Mansion Point&click adventure gegeben welche durch ein LLM from scratch erzeugt wurde:

Karte 1: Start
    Team-Wahl: Empfohlen: Syd oder Razor (Musik-Talent) + Bernard (Technik).
    Eingang: Gehe zum Haus. Unter der Fußmatte liegt der Türschlüssel.
    Eingangshalle: Benutze Schlüssel mit Haustür. Drinnen: Vorsicht vor Edna oder Weird Ed! Wenn sie kommen, schnell in einen Nebenraum flüchten.
    Küche: Gehe nach rechts durch die Tür. Nimm das Fleisch aus dem Kühlschrank und die Getränkedose (für später).

Karte 2: Das Gefängnis & Der Fluchtplan
    Trigger: Wenn ein Charakter geschnappt wird, landet er im Kerker.
    Ausbruch: Drücke den losen Stein an der Wand (links neben der Tür), um die Tür kurz zu öffnen.
    Taktik: Nutze einen Charakter als „Köder“, um die Edisons abzulenken, während die anderen zwei das Haus erkunden.
    Wichtig: Speichere oft! Ein falscher Schritt führt zum Game Over.

Karte 3: Die Vorräte im Erdgeschoss
    Esszimmer: Nimm die Früchte von der Schüssel.
    Speisekammer: Nimm den Glasschneider (hinter der Glasvitrine) und den Saft.
    Wohnzimmer: Benutze die Kassette mit dem Kassettenrekorder (wichtig für später, um das Glas zu zerschmettern).
    Bibliothek: Öffne die Lampe, um das Zimmer zu erhellen. Nimm den Loseblatt-Katalog.

Karte 4: Weird Eds Zimmer (1. Stock)
    Vorsicht: Wenn Ed dich sieht, fliegst du raus.
    Ablenkung: Warte, bis Ed sein Zimmer verlässt (um das Paket zu holen) oder nutze die Türklingel an der Haustür, um ihn runterzulocken.
    Gegenstand: Nimm den Hamster (nur als Razor/Syd relevant für böse Enden) und das Sparschwein. Brich das Schwein auf, um Geld zu erhalten.
    Schlüssel: Nimm den Schlüssel vom Nachttisch.

Karte 5: Das Musikzimmer & Die Glasplatte
    Vorbereitung: Geh ins Musikzimmer (1. Stock).
    Aktion: Benutze den Glasschneider mit der Vitrine, um die Kassette zu stehlen.
    Klavier: Wenn du Razor oder Syd bist, kannst du hier spielen, aber das Ziel ist die Schallplatte im Zimmer daneben.
    Tipp: Du musst die Schallplatte auf den Rekorder im Wohnzimmer legen und aufnehmen, um später das Fenster im Labor zu sprengen.

Karte 6: Das Schwimmbad (Außenbereich)
    Gefahr: Das Wasser ist radioaktiv! Nicht hineingehen.
    Aktion: Leere das Becken, indem du das Ventil im Keller (unter der Küche) benutzt.
    Fundstück: Am Boden des leeren Beckens liegt der leuchtende Schlüssel.
    Wichtig: Fülle das Becken sofort wieder auf, da sonst der Reaktor überhitzt (Game Over Timer startet!).

Karte 7: Der Grüne Tentakel
    Ort: 2. Stock, Weg zum Dachboden.
    Problem: Der Grüne Tentakel hat Hunger und lässt dich nicht vorbei.
    Lösung: Gib ihm den Obstteller aus dem Esszimmer und den Saft (oder die Cola).
    Ergebnis: Er wird dein Freund und lässt dich passieren. Du hast nun Zugang zu den oberen Räumen und dem Funkgerät.

Karte 8: Das Paket & Das Telefon
    Paket: Wenn die Türklingel läutet, renne mit einem Charakter zur Haustür und schnapp dir das Paket, bevor Ed es bekommt.
    Inhalt: Briefmarken.
    Telefon: Benutze das Telefon im Arbeitszimmer. Wähle die Nummer vom „Metzger“ (findest du im Loseblatt-Katalog), um Edna abzulenken.
    Nächster Schritt: Während Edna telefoniert, schleiche in ihr Zimmer, um den Schlüssel zum Labor zu finden.
    
Diese Anleitung gibt in natürlicher Sprache einen Ablauf vor um das Spiel erfolgreich zu spielen. Einziges Problem bei dieser Anleitung ist, dass es kein ausführbarer Computer code ist sondern an menschliche Leser adressiert wurde. In der DIKW pyramide ist die Komplettlösung also auf dem Layer 3 (knowledge) angesiedelt. Damit eine KI Maniac Mansion automatisiert durchspöielen kann, muss man die Anleitung auf eine niedrige DIKW Stufe übersetzen also auf Stufe 2 und Stufe 1 (Daten).

Sowas wird über ein Text to action model realisiert. DAs erhält eine Karteikarte als Input und erzeugt dafür die Mausbewegung als Ausgabe.

Hier die simulierten Mausbewegungen für Karteikarte #1 innerhalb der SCUMM-Engine bei einer Auflösung von 320x200 Pixeln. Das json file enthält dieselben Anweisungen wie die textuelle Komplettlösung auch nur mit dem Unterschied dass es nicht auf dem DIKW layer 3 sondern auf dem untersten Layer 1 angesiedelt ist. Als Folge gibt es numerische Koordinaten die definieren wo genau der Mauscursor hinbewegt wird.

{
  "card_id": 1,
  "title": "Start",
  "steps": [
    {
      "action_order": 1,
      "description": "Walk to the front door area",
      "command": "WALK_TO",
      "target_coords": {"x": 160, "y": 140},
      "wait_ms": 2000
    },
    {
      "action_order": 2,
      "description": "Pick up the door mat",
      "verb_click": {"x": 40, "y": 170, "label": "PICK_UP"},
      "object_click": {"x": 155, "y": 155, "label": "DOOR_MAT"},
      "wait_ms": 1500
    },
    {
      "action_order": 3,
      "description": "Pick up the key under the mat",
      "verb_click": {"x": 40, "y": 170, "label": "PICK_UP"},
      "object_click": {"x": 155, "y": 155, "label": "KEY"},
      "wait_ms": 1000
    },
    {
      "action_order": 4,
      "description": "Use key with front door",
      "verb_click": {"x": 80, "y": 180, "label": "USE"},
      "inventory_click": {"x": 300, "y": 170, "label": "KEY"},
      "object_click": {"x": 160, "y": 100, "label": "FRONT_DOOR"},
      "wait_ms": 3000
    },
    {
      "action_order": 5,
      "description": "Enter the house",
      "command": "WALK_TO",
      "target_coords": {"x": 160, "y": 90},
      "wait_ms": 2000
    },
    {
      "action_order": 6,
      "description": "Go to the kitchen (right door)",
      "command": "WALK_TO",
      "target_coords": {"x": 280, "y": 120},
      "wait_ms": 2500
    },
    {
      "action_order": 7,
      "description": "Open refrigerator",
      "verb_click": {"x": 40, "y": 180, "label": "OPEN"},
      "object_click": {"x": 100, "y": 100, "label": "REFRIGERATOR"},
      "wait_ms": 1000
    },
    {
      "action_order": 8,
      "description": "Pick up the meat",
      "verb_click": {"x": 40, "y": 170, "label": "PICK_UP"},
      "object_click": {"x": 105, "y": 110, "label": "MEAT"},
      "wait_ms": 1000
    }
  ]
}

Language parsing with a DIKW pyramid

In a role playing game there is an npc quest available which asks the human player to collect an item in the wood. To verify if the human player has fulfilled the quest, the game engine needs to understand the words which is equal to downwards in the DIKW pyramid. A sentence consists of single words, these words are associated with sprites in the game engine and also with location in the map. The association between an abstract word and the information in the game engine is stored in a DIKW pyramid which is a database.

A single sentence can be submitted to the dikw database and the database resolves the request so that it will become machine readable. Going upwards and downwards in the DIKW pyramid is equal to symbol grounding.

A DIKW pyramid consists of layers which are storing different sort of information. The lowest layer is accessible for a computer program and consists of location in a map, trajectories, sprites, tile maps and numerical color information. A possible entry might be [100,30] for a position in a map or (100,120,90) for a RGB color information.

On the next layer "information" a different sort of information are stored which are words. A word is a string which can be understand by a human but doesn't provide sense for a computer. For a human the word "wood" makes sense, but for a computer the same string is only an array of characters without any meaning. Its the task of the DIKW pyramid to link the word "wood" with a location in the map. The link allows the computer to resolve the meaning.

How to scale up artificial intelligence

 In the past it was mostly unknown how to create Artificial intelligence, even restricted problems like the game of chess or robot control in a warehouse were recognized as hard to realize in software. The cause was a missing understanding of the domain and missing tools to implement AI domains on a computer.

What is available today is a pathway how to realize Artificial intelligence in a step by step fashion. It has to do with splitting the task between human and computer. In case of computer chess the situation can be described. Automating the entire process of playing the game is difficult, but the computer can be used to only count the pieces on the board and print out the number to the screen. This information helps the human player to decide what for the best move. So the human sees the chessboard itself and a dashboard with important information generated by a computer.

The next task is to improve the dashboard, in a sense that more information are recognized by the computer software like the value for each piece, similar board situation found in the database, and the allowed possible moves. All the information are shown on the same dashboard and reduce the workload for the human operator. The resulting AI system can't be called a true AI but its only a semi-autonomous system.

Instead of answering the question how to play chess with a computer the modified task is to communicate between a human and a computer program about the domain of chess. This communication is realized with an algorithm. The human enters high level actions like "move pawn forward" or "protect queen" and the AI software is in charge to realize this command on the board.