Unity Gesture Recognizer #1 - Base Code

목차

---

https://github.com/richardlee-kr/DrawRecognition/tree/features/Recognition/Base

GitHub - richardlee-kr/DrawRecognition

 

RecognitionManager

전체적인 Recognition을 관리하는 class

Awake()

btn에 각각의 SetMode를 Listener로 추가

m_drawable의 OnDrawFinished에 RecognitionManager의 OnDrawFinished를 구독

현재 mode 초기화

SetMode(mode)

현재 mode를 변경하는 함수

m_drawable에 그려진 것을 모두 지움

바뀌는 mode에 따라 UI 활성화/비활성화

OnDrawFinished(points)

m_drawable의 OnDrawFinished가 호출되면 같이 호출되는 함수

TEMPLATE mode인 경우 그려진 points들을 m_templates에 전달

RECOGNITION mode인 경우 그려진 points들로 gesture recognition 실행 후 결과 출력

OnApplicationQuit()

Application이 종료될 때 m_template에 있는 templates들을 저장하도록 함

- m_drawable: Drawable

- m_recognitionPanel: RecognitionPanel

- m_templateReviewer: TemplateReviewer

- m_templates: GestureTemplates

- m_mode: RecognizerMode

- m_currentRecognizer: IRecognizer

 

- _templateName: string

 

- btn_recognizeMode: Button

- btn_templateMode: Button

- btn_reviewMode: Button

- txt_recognitionResult: TextMeshProUGUI

- input_templateName: TMP_InputField

---

+ SetMode(mode: RecognizerMode)

+ OnDrawFinished(points: DollarPoint[])

 

<<enumeration>>

RecognzierMode

  RECOGNITION
  TEMPLATE
  REVIEW

 

<<struct>>

GestureTemplate

+ name: string

+ points: DollarPoints[]

------------------------------

+ GestureTemplate(templateName: string, preparedPoints: DollarPoints[])

using System;
using UnityEngine;
using UnityEngine.UI;
using TMPro;

public class RecognitionManager : MonoBehaviour
{
    [SerializeField] private Drawable m_drawable;

    [SerializeField] private Button btn_recognizeMode;
    [SerializeField] private Button btn_templateMode;
    [SerializeField] private Button btn_reviewMode;
    [SerializeField] private TextMeshProUGUI txt_recognitionResult;
    [SerializeField] private TMP_InputField input_templateName;

    [SerializeField] private RecognitionPanel m_recogntionPanel;
    [SerializeField] private TemplateReviewer m_templateReviewer;

    private string _templateName => input_templateName.text;

    private GestureTemplates m_templates => GestureTemplates.Get();
    private IRecognizer m_currentRecognizer;
    [SerializeField]
    private RecognizerMode m_mode = RecognizerMode.TEMPLATE;

    public enum RecognizerMode
    {
        RECOGNITION,
        TEMPLATE,
        REVIEW,
    }

    [Serializable]
    public struct GestureTemplate
    {
        public string name;
        public DollarPoint[] points;

        public GestureTemplate(string templateName, DollarPoint[] preparedPoint)
        {
            name = templateName;
            points = preparedPoint;
        }
    }

    private void Awake()
    {
        btn_recognizeMode.onClick.AddListener(() => SetMode(RecognizerMode.RECOGNITION));
        btn_templateMode.onClick.AddListener(() => SetMode(RecognizerMode.TEMPLATE));
        btn_reviewMode.onClick.AddListener(() => SetMode(RecognizerMode.REVIEW));

        m_drawable.OnDrawFinished += OnDrawFinished;

        SetMode(m_mode);
    }

    private void SetMode(RecognizerMode mode)
    {
        m_mode = mode;

        m_drawable.ClearDrawing();

        input_templateName.gameObject.SetActive(mode == RecognizerMode.TEMPLATE);
        txt_recognitionResult.gameObject.SetActive(mode == RecognizerMode.RECOGNITION);

        m_drawable.gameObject.SetActive(mode != RecognizerMode.REVIEW);
        m_templateReviewer.SetVisibility(mode == RecognizerMode.REVIEW);
        m_recogntionPanel.gameObject.SetActive(mode == RecognizerMode.RECOGNITION);
    }

    private void OnDrawFinished(DollarPoint[] points)
    {
        if(m_mode == RecognizerMode.TEMPLATE)
        {
            //Save Template
            GestureTemplate newTemplate = new GestureTemplate(_templateName, points);
            GestureTemplate preparedTemplate = new GestureTemplate(_templateName, m_currentRecognizer.Normalize(points, 64));
            m_templates.RawTemplates.Add(newTemplate);
            m_templates.ProceedTemplates.Add(preparedTemplate);
        }
        else if(m_mode == RecognizerMode.RECOGNITION)
        {
        	if(m_currentRecognizer == null)
            	return;
            //Do Recognition
            (string, float) result = m_currentRecognizer.DoRecognition(points, 64, m_templates.RawTemplates);
            string resultText = "";
            switch (m_currentRecognizer)
            {

            }
            txt_recognitionResult.text = resultText;
        }
    }

    private void OnApplicationQuit()
    {
        m_templates.Save();
    }
}

Drawable

마우스입력에 따라 점들을 표시하는 class

Update()

마우스가 일정 범위내에 있다면 Draw를 호출

Draw()

마우스 입력에 따라 CreateBrush(), AddPoint() 등을 호출

특정 키를 누르면 OnDrawFinished()를 호출하고 그려진 것을 모두 지움

CreateBrsuh()

m_brush를 생성하고 m_lineRenderer를 할당

현재 마우스 위치를 lineRenderer의 시작점으로 설정

AddPoint(pointPos)

lineRenderer의 새로운 position을 추가

m_drawPoints에 TemplateDisplay의 크기에 맞춰 후처리한 좌표를 추가

InRange()

현재 마우스의 위치가 WIDTH와 HEIGHT 내에 있는지 확인하는 함수

ClearDrawing()

그려진 것을 모두 지우는 함수

m_drawPoints를 초기화하고 생성했던 brush들을 모두 삭제한다.

 

- m_camera: Camera

- m_brush: GameObject

- m_lineRenderer: LineRenderer

- m_drawPoints: List<DollarPoint>

- lastPos: Vector2

 

- WIDTH: float

- HEIGHT: float

 

- OnDrawFinished: event Action<DollarPoint[]>

---

- Draw()

- CreateBrush()

- AddPoint(pointPos: Vector2)

- InRange(): bool

+ ClearDrawing()

using System;
using System.Collections.Generic;
using UnityEngine;

public class Drawable : MonoBehaviour
{
    [SerializeField] private Camera m_camera;
    [SerializeField] private GameObject m_brush;

    [SerializeField] private float WIDTH = 10;
    [SerializeField] private float HEIGHT = 10;

    LineRenderer m_lineRenderer;
    Vector2 lastPos;

    private List<DollarPoint> m_drawPoints = new List<DollarPoint>();

    public event Action<DollarPoint[]> OnDrawFinished;

    private void Update()
    {
        if(InRange())
        {
            Draw();
        }
    }

    private void Draw()
    {
        if(Input.GetMouseButtonDown(0))
        {
            CreateBrush();
        }
        if(Input.GetMouseButton(0))
        {
            Vector2 mousePos = m_camera.ScreenToWorldPoint(Input.mousePosition);
            if(mousePos != lastPos)
            {
                AddPoint(mousePos);
                lastPos = mousePos; 
            }
        }
        if(Input.GetMouseButtonUp(0))
        {
            m_lineRenderer = null;
        }

        if(Input.GetKeyDown(KeyCode.Space))
        {
            Debug.Log("Finish");
            OnDrawFinished?.Invoke(m_drawPoints.ToArray());
            ClearDrawing();
        }
    }

    private void CreateBrush()
    {
        GameObject _brushInstance = Instantiate(m_brush, this.transform);
        m_lineRenderer = _brushInstance.GetComponent<LineRenderer>();

        Vector2 _mousePos = m_camera.ScreenToWorldPoint(Input.mousePosition);

        m_lineRenderer.SetPosition(0, _mousePos);
        m_lineRenderer.SetPosition(1, _mousePos);
    }


    private void AddPoint(Vector2 pointPos)
    {
        int _positionIndex = m_lineRenderer.positionCount;
        m_lineRenderer.positionCount++;
        m_lineRenderer.SetPosition(_positionIndex, pointPos);

        m_drawPoints.Add(new DollarPoint((pointPos.x+WIDTH/2)*100, (pointPos.y+HEIGHT/2)*100));
    }

    private bool InRange()
    {
        Vector2 mousePos = m_camera.ScreenToWorldPoint(Input.mousePosition);

        if (mousePos.x < transform.position.x - WIDTH / 2)
            return false;
        if (mousePos.x > transform.position.x + WIDTH / 2)
            return false;
        if (mousePos.y < transform.position.y - HEIGHT / 2)
            return false;
        if (mousePos.y > transform.position.y + HEIGHT / 2)
            return false;

        return true;
    }

    public void ClearDrawing()
    {
        m_drawPoints.Clear();
        foreach (Transform child in transform)
        {
            Destroy(child.gameObject);
        }
    }

    private void OnDrawGizmos()
    {
        Gizmos.color = Color.green;
        Gizmos.DrawWireCube(transform.position, new Vector3(WIDTH, HEIGHT, 0));
    }
}

FloatExtensions

Remap 함수를 정의하기 위한 class

public static class FloatExtensions
{
    public static float Remap(this float value, float from1, float to1, float from2, float to2)
    {
        return (value - from1) / (to1 - from1) * (to2 - from2) + from2;
    }

}

DollarPoint

알고리즘에 사용하기 위한 2D point를 저장하는 class

using System;
using UnityEngine;

[Serializable]
public struct DollarPoint
{
    public Vector2 point;

    public DollarPoint(float x, float y)
    {
        point = new Vector2(x, y);
    }
}

GestureTemplates

Gesture Template들을 runtime에서 저장하고 json 형태로 저장/불러오기 하는 class

GetTemplates()

저장된 모든 RawTemplates을 반환하는 함수

RemoveAtIndex(indexToRemove)

특정 index에 있는 RawTemplate를 삭제하는 함수

GetRawTemplatesByName(name)

특정 name을 가진 RawTemplates를 모두 반환하는 함수

Save()

RawTemplates에 저장되어 있는 template들을 json형태로 저장하는 함수

Load()

json으로 저장된 template들을 RawTemplates으로 불러오는 함수

 

<<static>>

- instance: GestureTemplates

- RawTemplates: List<RecognitionManager.GestureTemplate>()

 

---

 

+ GetTemplates(): List<RecognitionManager.GestureTemplate>()

+ RemoveAtIndex(indexToRemove: int)

+ GetRawTemplatesByName(name: string): RecognitionManager.GestureTemplate[]

+ Save()

+ Load()

using System.Collections.Generic;
using System.IO;
using System.Linq;
using UnityEngine;

public class GestureTemplates
{
    private static GestureTemplates instance;

    public static GestureTemplates Get()
    {
        if(instance == null)
        {
            instance = new GestureTemplates();
            instance.Load();
        }

        return instance;
    }
    public List<RecognitionManager.GestureTemplate> RawTemplates = new List<RecognitionManager.GestureTemplate>();
    //public List<RecognitionManager.GestureTemplate> ProceedTemplates = new List<RecognitionManager.GestureTemplate>();

    public List<RecognitionManager.GestureTemplate> GetTemplates()
    {
        //return ProceedTemplates;
        return RawTemplates;
    }

    public void RemoveAtIndex(int indexToRemove)
    {
        //ProceedTemplates.RemoveAt(indexToRemove);
        RawTemplates.RemoveAt(indexToRemove);
    }

    public RecognitionManager.GestureTemplate[] GetRawTemplatesByName(string name)
    {
        return RawTemplates.Where(template => template.name == name).ToArray();
    }

    public void Save()
    {
        string path = Application.persistentDataPath + "/SavedTemplates.json";
        string potion = JsonUtility.ToJson(this);
        File.WriteAllText(path, potion);
    }

    private void Load()
    {
        string path = Application.persistentDataPath + "/SavedTemplates.json";
        if (File.Exists(path))
        {
            GestureTemplates data = JsonUtility.FromJson<GestureTemplates>(File.ReadAllText(path));
            RawTemplates.Clear();
            RawTemplates.AddRange(data.RawTemplates);
            //ProceedTemplates.Clear();
            //ProceedTemplates.AddRange(data.ProceedTemplates);
        }
    }

}

IRecognizer

Recognizer에 들어갈 inferace

using System.Collections.Generic;

public interface IRecognizer
{
    public DollarPoint[] Normalize(DollarPoint[] points, int n);

    public (string, float) DoRecognition(DollarPoint[] points, int n, 
        List<RecognitionManager.GestureTemplate> gestureTemplates);
}

Recognizer

여러가지 Recognizer의 Base class

public class Recognizer
{
}

DisplayTemplate

GestureTemplate의 정보를 가지고 texture의 pixel을 바꾸어 화면에 표시하는 class

Awake()

Clear() 함수 호출

ColorBetween(startPos, endPos, width, color)

startPos부터 endPos까지 점들에 대해서 MarkPixelsToColor 호출

MarkPixelsToColor(centerPixel, width, color)

width에 따라 주변 pixel들에 대해서 MarkPixelsToChange 호출

MarkPixelsToChange(x, y, color, textureColors)

array 형태로 저장되어 있는 textureColors에 color를 적용

ApplyMarkedPixelChanges(texture, colors)

texture의 pixel들을 color로 변경

Draw(template)

drawable_texture 초기화

drawable_texture의 현재 color를 cur_color에 저장

template에 저장된 점들을 Array로 만들기

두 점을 잇도록 ColorBetween 함수 호출

drawable_texture에 cur_color를 적용

Clear()

drawable_texture의 모든 점을 white로 설정

- drawable_texture: Texture2D

- cur_colors: Color32[]

---

 

- ColorBetween(startPos: Vector2, endPos: Vector2, width: int, color: Color)

- MarkPixelsToColor(centerPixel: Vector2, width: int, color: Color)

- MarkPixelsToChange(x: int, y: int, color: Color, textureColors: Color32[])

- ApplyMarkedPixelChanges(texture: Texture2D, colors: Color32[])

 

+ Draw(template: RecognitionManager.GestureTemplate)

+ Clear()

using System.Linq;
using UnityEngine;

public class DisplayTemplate : MonoBehaviour
{
    [SerializeField] private Texture2D drawable_texture;

    private Color32[] cur_colors;

    private void Awake()
    {
        Clear();
    }


    public void Draw(RecognitionManager.GestureTemplate template)
    {
        Clear();

        cur_colors = drawable_texture.GetPixels32();

        DollarPoint[] points = template.points.Distinct().ToArray();

        for (int i = 1; i < points.Length; i++)
        {
            Vector2 prev = points[i - 1].point;
            Vector2 curr = points[i].point;

            ColorBetween(prev, curr, 2, Color.red);
        }

        ApplyMarkedPixelChanges(drawable_texture, cur_colors);
    }

    public void Clear()
    {
        Color[] clean_colours_array = new Color[(int)drawable_texture.width * (int)drawable_texture.height];
        for (int x = 0; x < clean_colours_array.Length; x++)
            clean_colours_array[x] = Color.white;

        drawable_texture.SetPixels(clean_colours_array);
        drawable_texture.Apply();

    }

    private void ColorBetween(Vector2 startPos, Vector2 endPos, int width, Color color)
    {
        float distance = Vector2.Distance(startPos, endPos);

        Vector2 cur_position = startPos;

        float lerp_steps = 1 / distance;

        for (float lerp = 0; lerp <= 1; lerp += lerp_steps)
        {
            cur_position = Vector2.Lerp(startPos, endPos, lerp);
            MarkPixelsToColor(cur_position, width, color);
        }
    }

    private void MarkPixelsToColor(Vector2 centerPixel, int width, Color color)
    {
        int centerX = (int)centerPixel.x;
        int centerY = (int)centerPixel.y;

        for(int x = centerX - width; x <= centerX + width; x++)
        {
            if(x >= (int)drawable_texture.width || x < 0)
            {
                continue;
            }

            for(int y = centerY - width; y <= centerY + width; y++)
            {
                MarkPixelsToChange(x, y, color, cur_colors);
            }
        }
    }

    private void MarkPixelsToChange(int x, int y, Color color, Color32[] textureColors)
    {
        int array_pos = y * (int)drawable_texture.width + x;

        if(array_pos > textureColors.Length || array_pos < 0)
        {
            return;
        }

        textureColors[array_pos] = color;
    }

    private void ApplyMarkedPixelChanges(Texture2D texture, Color32[] colors)
    {
        texture.SetPixels32(colors);
        texture.Apply(false);
    }
}

 

TemplateReviewer

GestureTemplates에 저장된 RawTemplates를 확인할 수 있도록 하는 UI class

Awake()

UI들에 listener를 할당

UpdateTemplates() 호출

SetVisibility(visible)

this와 m_display를 활성화/비활성화

만약 활성화한다면 UpdateDisplay()와 ChooseTemplateToShow(0) 호출

UpdateDisplay()

UpdateTemplates() 호출

m_currentTemplateName을 사용하여 해당 name을 가진 모든 template을 가져오기

만약 template이 존재한다면 m_display의 Draw()를 호출하여 현재 template 그리기

아니라면 m_display의 Clear() 호출

UpdateTemplates()

m_templateNames에 m_templates의 RawTemplates에 저장된 모든 template의 이름을 가져오기

dd_templateNameList의 option에 모든 template의 이름 추가

만약 존재한다면 button들과 dropdown 활성화

ChooseTemplateIndex(increament)

이전/다음 template을 보이기

이전/다음 template이 존재한다면 UpdateDisplay() 호출

RemoveTemplate()

m_currentTemplateName을 가지는 RawTemplates들을 IEnumerable로 가져오기

삭제할 template을 m_currentTemplateIndex로 가져오기

삭제할 template의 index을 RawTemplate에서 찾기

해당 index를 삭제

 

template이 남아있다면 m_currentTemplateIndex--

template이 다 삭제되었다면 0번째 template 선택

 

UpdateDipslay() 호출

ChooseTemplateToShow(index)

0번째 template을 선택

UpdateDisplay() 호출

- m_templates: GestureTemplates

- m_templateNames: List<string>

- m_currentTemplateName: string

- m_currentTemplateIndex: int

 

- m_display: DisplayTemplate

- dd_templateNameList: TMP_Dropdown

- btn_previous: Button

- btn_ next: Button

- btn_remove: Button

 

---

- UpdateDisplay()

- UpdateTemplates()

- ChooseTemplateIndex(increament: int)

- RemoveTemplate()

- ChooseTemplateToShow(index: int)

+ SetVisibility(visible: bool)

using System.Collections.Generic;
using UnityEngine;
using TMPro;
using UnityEngine.UI;
using System.Linq;

public class TemplateReviewer : MonoBehaviour
{
    [SerializeField] private DisplayTemplate m_display;

    [SerializeField] private TMP_Dropdown dd_templateNameList;
    [SerializeField] private Button btn_previous;
    [SerializeField] private Button btn_next;
    [SerializeField] private Button btn_remove;

    private List<string> m_templateNames;
    private string m_currentTemplateName;
    private int m_currentTemplateIndex;

    private GestureTemplates m_templates => GestureTemplates.Get();

    private void Awake()
    {
        btn_previous.onClick.AddListener(() => ChooseTemplateIndex(-1));
        btn_next.onClick.AddListener(() => ChooseTemplateIndex(1));
        btn_remove.onClick.AddListener(() => RemoveTemplate());

        dd_templateNameList.onValueChanged.AddListener(ChooseTemplateToShow);

        UpdateTemplates();
    }

    public void SetVisibility(bool visible)
    {
        this.gameObject.SetActive(visible);
        m_display.gameObject.SetActive(visible);
        if(visible)
        {
            UpdateDisplay();
            ChooseTemplateToShow(0);
        }
    }

    private void UpdateDisplay()
    {
        UpdateTemplates();
        RecognitionManager.GestureTemplate[] _templates = m_templates.GetRawTemplatesByName(m_currentTemplateName);
        if(_templates.Length > 0)
        {
            m_display.Draw(_templates[m_currentTemplateIndex]);
        }
        else
        {
            m_display.Clear();
        }
    }

    private void UpdateTemplates()
    {
        m_templateNames = m_templates
            .RawTemplates
            .Select(templates => templates.name)
            .Distinct().ToList();

        dd_templateNameList.options = m_templateNames
            .Select(templateName => new TMP_Dropdown.OptionData(templateName))
            .ToList();

        bool anyTemplateAvailable = m_templateNames.Any();
        btn_remove.gameObject.SetActive(anyTemplateAvailable);
        btn_previous.gameObject.SetActive(anyTemplateAvailable);
        btn_next.gameObject.SetActive(anyTemplateAvailable);
        dd_templateNameList.gameObject.SetActive(anyTemplateAvailable);
    }

    private void ChooseTemplateIndex(int increament)
    {
        m_currentTemplateIndex += increament;
        if (m_currentTemplateIndex < 0 || m_currentTemplateIndex > m_templates.GetRawTemplatesByName(m_currentTemplateName).Length - 1)
        {
            m_currentTemplateIndex -= increament;
            return;
        }
        else
        {
            UpdateDisplay();
        }
    }

    private void RemoveTemplate()
    {
        //IEnumerable<RecognitionManager.GestureTemplate> templatesByName = m_templates.ProceedTemplates
        //    .Where(template => template.name == m_currentTemplateName).ToList();
        IEnumerable<RecognitionManager.GestureTemplate> templatesByName = m_templates.RawTemplates
            .Where(template => template.name == m_currentTemplateName).ToList();
        Debug.Log(templatesByName.Count());
        RecognitionManager.GestureTemplate templateToRemove = templatesByName
            .ElementAt(m_currentTemplateIndex);

        //int indexToRemove = m_templates.ProceedTemplates.IndexOf(templateToRemove);
        int indexToRemove = m_templates.RawTemplates.IndexOf(templateToRemove);
        m_templates.RemoveAtIndex(indexToRemove);

        if(m_currentTemplateIndex != 0)
        {
            m_currentTemplateIndex--;
        }

        if(templatesByName.Count() == 1)
        {
            ChooseTemplateToShow(0);
        }

        UpdateDisplay();
    }

    private void ChooseTemplateToShow(int index)
    {
        string chosenTemplateName = m_templateNames[index];
        m_currentTemplateName = chosenTemplateName;
        m_currentTemplateIndex = 0;
        UpdateDisplay();
    }
}

RecognitionPanel

Recogntion Algorithm을 선택하는 UI class

using System.Collections;
using System.Collections.Generic;
using TMPro;
using UnityEngine;

public class RecognitionPanel : MonoBehaviour
{
    [SerializeField] private TMP_Dropdown dd_algorithmList;
}

 

Ref.

https://github.com/kasiula03/2D-Gesture-Recognition

GitHub - kasiula03/2D-Gesture-Recognition