Right angle symbol

Using a \newcommand macro to mark a right angle between three points:

\usetikzlibrary{calc}
\newcommand\rightAngle[4]{
  \pgfmathanglebetweenpoints{\pgfpointanchor{#2}{center}}{\pgfpointanchor{#1}{center}}
  \coordinate (tmpRA) at ($(#2)+(\pgfmathresult+45:#4)$);
  \draw[blue!80!black,thick] ($(#2)!(tmpRA)!(#1)$) -- (tmpRA) -- ($(#2)!(tmpRA)!(#3)$);
}
\begin{tikzpicture}[scale=3]
  \coordinate (O) at (0,0);
  \coordinate (X) at (2,0);
  \coordinate (Y) at (0,2);
  \draw[thick] (X) -- (O) -- (Y);
  \rightAngle{X}{O}{Y}{0.6}
\end{tikzpicture}

Using a \pic macro with a predefined input format:

\usetikzlibrary{calc}
\tikzset{
  pics/right angle/.style args={(#1)-(#2)-(#3):#4}{
    code={
      \pgfmathanglebetweenpoints{\pgfpointanchor{#2}{center}}{\pgfpointanchor{#1}{center}}
      \coordinate (tmpRA) at ($(#2)+(\pgfmathresult+45:#4)$);
      \draw[blue!80!black,thick] ($(#2)!(tmpRA)!(#1)$) -- (tmpRA) -- ($(#2)!(tmpRA)!(#3)$);
    }
  }
}
\begin{tikzpicture}[scale=3]
  \coordinate (O) at (0,0);
  \coordinate (X) at (2,0);
  \coordinate (Y) at (0,2);
  \draw[thick] (X) -- (O) -- (Y);
  \pic {right angle={(X)-(O)-(Y):0.6}};
\end{tikzpicture}

An alternative, shorter \def macro by Alexandros Tsagkaropolulos is the following:

\usetikzlibrary{calc}
\def\MarkRightAngle[size=#1](#2,#3,#4){
  \draw[blue!80!black,thick] ($(#3)!#1!(#2)$) -- ($($(#3)!#1!(#2)$)!#1!90:(#2)$) -- ($(#3)!#1!(#4)$)
}
\begin{tikzpicture}[scale=3]
  \coordinate (O) at (0,0);
  \coordinate (X) at (2,0);
  \coordinate (Y) at (0,2);
  \draw[thick] (X) -- (O) -- (Y);
  \MarkRightAngle[size=15pt](X,O,Y);
\end{tikzpicture}

Edit and compile if you like:

% Author: Izaak Neutelings (June 2020)
\documentclass[border=3pt,tikz]{standalone}
\usetikzlibrary{calc}
\tikzset{>=latex} % for LaTeX arrow head

\newcommand\rightAngle[4]{
  \pgfmathanglebetweenpoints{\pgfpointanchor{#2}{center}}{\pgfpointanchor{#3}{center}}
  \coordinate (tmpRA) at ($(#2)+(\pgfmathresult+45:#4)$);
  \draw[blue!80!black] ($(#2)!(tmpRA)!(#1)$) -- (tmpRA) -- ($(#2)!(tmpRA)!(#3)$);
  %\fill[red] (tmpRA) circle(0.02);
}

\begin{document}


\begin{tikzpicture}
  \coordinate (O) at (0,0);
  \coordinate (X) at (1,0);
  \coordinate (Y) at (0,1);
  \draw (X) -- (O) -- (Y);
  \rightAngle{Y}{O}{X}{0.40}
\end{tikzpicture}


\begin{tikzpicture}
  \def\ang{20}
  \coordinate (O) at (0,0);
  \coordinate (X) at (\ang:1);
  \coordinate (Y) at (\ang+90:1);
  \draw (X) -- (O) -- (Y);
  \rightAngle{Y}{O}{X}{0.40}
  \node[scale=0.7] at (45:1.4) {$\alpha=\pgfmathresult$};
\end{tikzpicture}


\begin{tikzpicture}
  \def\ang{20}
  \def\R{1.2}
  \coordinate (O) at (0,0);
  \coordinate (R) at (\ang:\R);
  \pgfmathanglebetweenpoints{\pgfpointanchor{O}{center}}{\pgfpointanchor{R}{center}}
  \draw (O) -- (R);
  \node[scale=0.7] at (45:1.4) {$\alpha=\pgfmathresult$};
  \coordinate (RA) at ($(O)+(\pgfmathresult+45:0.5*\R)$);
  \draw[dashed,very thin] (O) -- (RA);
  \draw[blue] ($(O)!(RA)!(R)$) -- (RA);
  \fill[red] (RA) circle(0.02);
\end{tikzpicture}


\begin{tikzpicture}
  \def\ang{30}
  \def\R{1.2}
  \coordinate (O) at (0,0);
  \coordinate (R) at (\ang:\R);
  \pgfmathanglebetweenpoints{\pgfpointanchor{O}{center}}{\pgfpointanchor{R}{center}}
  \draw (O) -- (R);
  \coordinate (RA) at ($(R)+(\pgfmathresult+90:0.3*\R)$);
  \coordinate (RA) at ($(R)+(\pgfmathresult+90:0.3*\R)$);
  \draw[dashed,very thin] (O) -- (RA);
  \draw[blue] ($(O)!(RA)!(R)$) -- (RA);
  \fill[red] (RA) circle(0.02);
\end{tikzpicture}


\end{document}