Picking up objects with stylus using Unity

0 votes
asked Nov 3, 2015 by nrraman929 (164 points)
Is there any documentation or samples on how to pick up an object with zSpace in an application made with Unity? The examples I found for the stylus seem to not provide examples for moving objects nor picking up objects

1 Answer

+1 vote
answered Nov 3, 2015 by dtwilleagerinfinitez-com (275 points)
We are working on some new samples that show these types of interactions, but they are not done yet.  Until then, you can check out the script below.  It has some corner case that don't work, but it does show the basic logic and math for manipulating objects directly.  It will allow you to pick up any object that has a collider associated with it.  It should work with Unity 4.x

using UnityEngine;

public class StylusClickAndDrag : MonoBehaviour
{
  void Start()
  {
    m_core = GameObject.FindObjectOfType(typeof(ZSCore)) as ZSCore;

    //Shader shader = Shader.Find("Hidden/Internal-Colored");
    //m_lineMaterial = new Material(shader);
    m_lineMaterial = new UnityEngine.Material("Shader \"Lines/Colored Blended\" {" +
  "SubShader { Pass { " +
  "    Blend Off " +
  "    ZWrite Off Cull Off Fog { Mode Off } " +
  "    BindChannels {" +
  "      Bind \"vertex\", vertex Bind \"color\", color }" +
  "} } }");
    m_lineMaterial.hideFlags = HideFlags.HideAndDontSave;
    // Turn on alpha blending
    m_lineMaterial.SetInt("_SrcBlend", (int)UnityEngine.Rendering.BlendMode.SrcAlpha);
    m_lineMaterial.SetInt("_DstBlend", (int)UnityEngine.Rendering.BlendMode.OneMinusSrcAlpha);
    // Turn backface culling off
    m_lineMaterial.SetInt("_Cull", (int)UnityEngine.Rendering.CullMode.Off);
    // Turn off depth writes
    m_lineMaterial.SetInt("_ZWrite", 0);
  }

  public void Update()
  {
    if (m_core != null)
    {
      Matrix4x4 stylusPose = m_core.GetTrackerTargetWorldPose(ZSCore.TrackerTargetType.Primary);
      m_stylusStartPosition = stylusPose.MultiplyPoint(Vector3.zero);
      Vector3 stylusDirection = stylusPose.MultiplyVector(Vector3.forward).normalized;
      m_stylusEndPosition = m_stylusStartPosition + stylusDirection * m_stylusLength;

      Ray ray = new Ray(m_stylusStartPosition, stylusDirection);
      RaycastHit[] hits = Physics.RaycastAll(ray);
      Vector3 stylusVirtualEnd = m_stylusStartPosition + stylusDirection * m_stylusLength;

      if (m_isDragging)
      {
        Quaternion rotation = UnityEngine.Quaternion.LookRotation(stylusPose.GetColumn(2), stylusPose.GetColumn(1));
        Quaternion newRotation = rotation * m_startRotation;

        Matrix4x4 matrix = Matrix4x4.TRS(Vector3.zero, newRotation, Vector3.one);
        Vector3 offset = matrix.MultiplyPoint(m_startOffset);

        m_selectedObject.transform.position = offset + stylusVirtualEnd;
        m_selectedObject.transform.rotation = newRotation;

        bool buttonPressed = m_core.IsTrackerTargetButtonPressed(ZSCore.TrackerTargetType.Primary, 0);
        if (m_buttonPressed != buttonPressed)
        {
          // State Changed into button release
          if (m_buttonPressed)
          {
            // If we are still selecting the object when the button was pressed, signal a click.
            if (m_objectSelectedWhenPressed == m_selectedObject)
            {
              setObjectColor(Color.green);
              m_objectSelectedWhenPressed = null;
              m_isDragging = false;
            }
          }
        }
      }
      else
      {
        UnityEngine.Debug.Log("Hits: " + hits.Length);
        if (hits != null && hits.Length > 0)
        {
          foreach (RaycastHit hit in hits)
          {
            if (m_selectedObject == null)
            {
              m_selectedObjectColor = getObjectColor(hit.collider.gameObject);
            }
            m_selectedObject = hit.collider.gameObject;
            m_stylusLength = (hit.point - m_stylusStartPosition).magnitude;
            setObjectColor(Color.red);
            break;
          }
        }
        else
        {
          if (m_selectedObject != null)
          {
            setObjectColor(m_selectedObjectColor);
          }
          m_selectedObject = null;
          m_stylusLength = 3.0f;
        }

        // Find the state of the stylus buttons
        bool buttonPressed = m_core.IsTrackerTargetButtonPressed(ZSCore.TrackerTargetType.Primary, 0);
        if (m_buttonPressed != buttonPressed)
        {
          // State Changed into button release
          if (m_buttonPressed)
          {
            // If we are still selecting the object when the button was pressed, signal a click.
            if (m_objectSelectedWhenPressed == m_selectedObject)
            {
              setObjectColor(Color.green);
              m_objectSelectedWhenPressed = null;
              m_isDragging = false;
            }
          }
          else
          {
            // Got press event - save who is selected.
            m_objectSelectedWhenPressed = m_selectedObject;
            m_isDragging = true;

            Quaternion quat = Quaternion.Inverse(m_objectSelectedWhenPressed.transform.rotation);
            Matrix4x4 matrix = Matrix4x4.TRS(Vector3.zero, quat, Vector3.one);
            Vector3 offset = m_objectSelectedWhenPressed.transform.position - stylusVirtualEnd;
            m_startOffset = matrix.MultiplyPoint(offset);

            Quaternion rotation = UnityEngine.Quaternion.LookRotation(stylusPose.GetColumn(2), stylusPose.GetColumn(1));
            quat = Quaternion.Inverse(rotation);
            m_startRotation = quat * m_objectSelectedWhenPressed.transform.rotation;
          }
          m_buttonPressed = buttonPressed;
        }
      }
    }

  }

  public void setObjectColor(Color color)
  {
    Renderer renderer = m_selectedObject.GetComponent<Renderer>();
    Material material = renderer.material;
    material.SetColor("_Color", color);
  }

  public Color getObjectColor(GameObject gameObject)
  {
    Renderer renderer = gameObject.GetComponent<Renderer>();
    Material material = renderer.material;
    return material.GetColor("_Color");
  }

  public void OnRenderObject()
  {
    m_lineMaterial.SetPass(0);
    GL.PushMatrix();

    GL.Begin(GL.LINES);

    Color color = new Color(1.0f, 1.0f, 1.0f);
    GL.Color(color);
    GL.Vertex3(m_stylusStartPosition.x, m_stylusStartPosition.y, m_stylusStartPosition.z);

    GL.Color(color);
    GL.Vertex3(m_stylusEndPosition.x, m_stylusEndPosition.y, m_stylusEndPosition.z);

    GL.End();
    GL.PopMatrix();
  }

  private ZSCore m_core = null;
  private Vector3 m_stylusStartPosition = new Vector3();
  private Vector3 m_stylusEndPosition = new Vector3();
  private float m_stylusLength = 3.0f;
  private Material m_lineMaterial = null;
  private GameObject m_selectedObject = null;
  private Color m_selectedObjectColor = new Color();
  private GameObject m_objectSelectedWhenPressed = null;
  private bool m_buttonPressed = false;
  private bool m_isDragging = false;
  private Vector3 m_startOffset = new Vector3();
  private Quaternion m_startRotation = new Quaternion();
}
commented Nov 6, 2015 by nav (161 points)
This helped a lot! I was wondering if you know if there is anyway to replicate depth perception, such as there is on the zSpace Experience Application? I want to be able to move an object closer or farther by moving the stylus closer or farther from the screen.
commented Nov 10, 2015 by dtwilleagerinfinitez-com (275 points)
To understand how to set up a scene like zSpace Experience you should review http://developer.zspace.com/docs/aesthetics/

With those guidelines, and knowing that, by default, zSpace maps the virtual to physical space exactly 1 to 1, you can create an environment like zSpace Experience.
...