First, note that $\sin^2(tx)=\frac12(1-\cos(2tx))$. Hence, we see that
$$I(t)=\frac\pi4-\frac12 \int_0^\infty \frac{\cos(2tx)}{x^2+1}\,dx\tag1$$
Differentiating under the integral in $(1)$ can be justified by noting that the integral $\int_0^\infty \frac{x\sin(2tx)}{x^2+1}\,dx$ converges uniformly for $|t|\ge \delta>0$. Proceeding reveals
$$\begin{align}
I'(t)&=\int_0^\infty \frac{x\sin(2tx)}{x^2+1}\,dx\\\\
&=\int_0^\infty \frac{(x^2+1-1)\sin(2tx)}{x(x^2+1)}\,dx\\\\
&=\int_0^\infty \frac{\sin(2tx)}{x}\,dx-\int_0^\infty \frac{\sin(2tx)}{x(x^2+1)}\,dx\\\\
&=\frac\pi2 \text{sgn}(t)-\int_0^\infty \frac{\sin(2tx)}{x(x^2+1)}\,dx\tag2
\end{align}$$
Similarly, we can differentiate $(2)$ to obtain
$$\begin{align}
I''(t)&=-2\int_0^\infty \frac{\cos(2tx)}{x^2+1}\,dx\\\\
&=4I(t)-\pi\tag3
\end{align}$$
From $(3)$ we have $I''(t)-4I(t)=-\pi$, while from $(1)$ we see that $I(0)=0$ and from $(2)$ we see that $\lim_{t\to 0^\pm}I'(t)=\pm \frac\pi2$. Solving this ODE with these initial conditions, we find
$$I(t)=\frac\pi4 -\frac\pi4 e^{-2|t|}$$
On the path of Zacky, the missing part...
Let,
\begin{align}I&=\int_0^{\frac{\pi}{2}}x^2\sqrt{\tan x}\,dx\\
J&=\int_0^{\frac{\pi}{2}}\frac{x^2}{\sqrt{\tan x}}\,dx\\
\end{align}
Perform the change of variable $y=\sqrt{\tan x}$,
\begin{align}I&=\int_0^{\infty}\frac{2x^2\arctan^2\left(x^2\right)}{1+x^4}\,dx\\\\
J&=\int_0^{\infty}\frac{2x^2\arctan^2\left(\frac{1}{x^2}\right)}{1+x^4}\,dx\\
\end{align}
\begin{align}
\text{I+J}&=\int_0^{\infty}\frac{2x^2\left(\arctan\left(x^2\right)+\arctan\left(\frac{1}{x^2}\right)\right)^2}{1+x^4}\,dx-4\int_0^{\infty}\frac{x^2\arctan\left(x^2\right)\arctan\left(\frac{1}{x^2}\right)}{1+x^4}\,dx\\
&=\frac{\pi^2}{4}\int_0^{\infty}\frac{2x^2}{1+x^4}\,dx-4\int_0^{\infty}\frac{x^2\arctan\left(x^2\right)\arctan\left(\frac{1}{x^2}\right)}{1+x^4}\,dx\\
\end{align}
Perform the change of variable $y=\dfrac{1}{x}$,
\begin{align}
\text{K}&=\int_0^{\infty}\frac{2x^2}{1+x^4}\,dx\\
&=\int_0^{\infty}\frac{2}{1+x^4}\,dx\\
\end{align}
Therefore,
\begin{align}
\text{2K}=\int_0^{\infty}\frac{2\left(1+\frac{1}{x^2}\right)}{\left(x-\frac{1}{x}\right)^2+2}\,dx
\end{align}
Perform the change of variable $y=x-\dfrac{1}{x}$,
\begin{align}\text{2K}&=2\int_{-\infty}^{+\infty}\frac{1}{2+x^2}\,dx\\
&=2\left[\frac{1}{\sqrt{2}}\arctan\left(\frac{x}{\sqrt{2}}\right)\right]_{-\infty}^{+\infty}\\
&=2\times \frac{\pi}{\sqrt{2}}
\end{align}
therefore,
\begin{align}
\text{I+J}&=\frac{\pi^3}{4\sqrt{2}}-4\int_0^{\infty}\frac{x^2\arctan\left(x^2\right)\arctan\left(\frac{1}{x^2}\right)}{1+x^4}\,dx\\
\end{align}
Let $a>0$,
\begin{align}
\text{K}_1(a)&=\int_0^{\infty}\frac{x^2}{a+x^4}\,dx\\
&=\frac{1}{a}\int_0^{\infty}\frac{x^2}{1+\left(a^{-\frac{1}{4}}x\right)^4}\,dx\\
\end{align}
Perform the change of variable $y=a^{-\frac{1}{4}}x$,
\begin{align}
\text{K}_1(a)&=a^{-\frac{1}{4}}\int_0^{\infty}\frac{x^2}{1+x^4}\,dx\\
&=\frac{a^{-\frac{1}{4}}\pi}{2\sqrt{2}}
\end{align}
In the same manner,
\begin{align}
\text{K}_2(a)&=\int_0^{\infty}\frac{x^2}{1+ax^4}\,dx\\
&=\frac{a^{-\frac{3}{4}}\pi}{2\sqrt{2}}
\end{align}
Since, for $a$ real,
\begin{align}\arctan a=\int_0^1 \frac{a}{1+a^2t^2}\,dt\end{align}
then,
\begin{align}\text{L}&=\int_0^{\infty}\frac{x^2\arctan\left(x^2\right)\arctan\left(\frac{1}{x^2}\right)}{1+x^4}\,dx\\
&=\int_0^{\infty}\left(\int_0^1 \int_0^1 \frac{x^2}{(1+u^2x^4)\left(1+\frac{v^2}{x^4}\right)(1+x^4)}\,du\,dv\right)\,dx\\
&=\\
&\int_0^{\infty}\left(\int_0^1\int_0^1 \left(\frac{x^2}{(1-u^2)(1-v^2)(1+x^4)}-\frac{x^2}{1-u^2v^2}\left(\frac{u^2}{(1-u^2)(1+u^2x^4)}+\frac{v^2}{(1-v^2)(v^2+x^4)}\right)
\right)dudv\right)dx\\
&=\int_0^1\int_0^1 \left(\frac{\pi}{2\sqrt{2}(1-u^2)(1-v^2)}-\frac{1}{1-u^2v^2}\left(\frac{u^2\text{K}_2(u^2)}{1-u^2}+\frac{v^2\text{K}_1(v^2)}{1-v^2}\right)\right)dudv\\
&=\frac{\pi}{2\sqrt{2}}\int_0^1\int_0^1 \left(\frac{1}{(1-u^2)(1-v^2)}-\frac{1}{(1-u^2v^2)}\left(\frac{u^{\frac{1}{2}}}{1-u^2}+\frac{v^{\frac{3}{2}}}{1-v^2}\right)\right)dudv\\
&=\pi\int_0^1\left[\frac{\sqrt{v}\left(\text{ arctanh}\left(\sqrt{uv}\right)-\text{ arctan}\left(\sqrt{uv}\right)-\text{ arctanh}\left(uv\right)\right)+\arctan\left(\sqrt{u}\right)+\ln\left(\frac{\sqrt{1+u}}{1+\sqrt{u}}\right)}{2\sqrt{2}(1-v^2)}\right]_{u=0}^{u=1}\,dv\\
&=\frac{\pi}{2\sqrt{2}}\int_0^1\frac{\sqrt{v}\big(\text{ arctanh}\left(\sqrt{v}\right)-\text{ arctan}\left(\sqrt{v}\right)-\text{ arctanh}\left(v\right)\big)+\frac{\pi}{4}-\frac{1}{2}\ln 2}{1-v^2}\,dv\\
&=\frac{\pi}{2\sqrt{2}}\int_0^1\frac{\sqrt{v}\arctan\left(\frac{1-\sqrt{v}}{1+\sqrt{v}}\right)}{1-v^2}\,dv+\frac{\pi}{2\sqrt{2}}\left(\frac{\pi}{4}-\frac{1}{2}\ln 2\right)\int_0^1 \frac{1-\sqrt{v}}{1-v^2}\,dv+\\
&\frac{\pi}{2\sqrt{2}}\int_0^1\frac{\sqrt{v}\ln\left(\frac{1+\sqrt{v}}{2}\right)}{1-v^2}\,dv-\frac{\pi}{4\sqrt{2}}\int_0^1\frac{\sqrt{v}\ln\left(\frac{1+v}{2}\right)}{1-v^2}\,dv
\end{align}
Perform the change of variable $y=\dfrac{1-\sqrt{v}}{1+\sqrt{v}}$,
\begin{align}\text{R}_1&=\int_0^1\frac{\sqrt{v}\arctan\left(\frac{1-\sqrt{v}}{1+\sqrt{v}}\right)}{1-v^2}\,dv\\
&=\frac{1}{2}\int_0^1 \frac{(1-v)^2\arctan v}{v(1+v^2)}\,dv\\
&=\frac{1}{2}\int_0^1 \frac{\arctan v}{v}\,dv-\int_0^1 \frac{\arctan v}{1+v^2}\,dv\\
&=\frac{1}{2}\text{G}-\frac{1}{2}\Big[\arctan^2 v\Big]_0^1\\
&=\frac{1}{2}\text{G}-\frac{\pi^2}{32}\\
\text{R}_2&=\int_0^1 \frac{1-\sqrt{v}}{1-v^2}\,dv\\
&=\left[\ln\left(\frac{\sqrt{1+v}}{1+\sqrt{v}}\right)+\arctan\left(\sqrt{v}\right)\right]_0^1\\
&=\frac{\pi}{4}-\frac{1}{2}\ln 2\\
\end{align}
Perform the change of variable $y=\dfrac{1-\sqrt{v}}{1+\sqrt{v}}$,
\begin{align}\text{R}_3&=\int_0^1\frac{\sqrt{v}\ln\left(\frac{1+\sqrt{v}}{2}\right)}{1-v^2}\,dv\\
&=-\frac{1}{2}\int_0^1\frac{(1-v)^2\ln(1+v)}{v(1+v^2)}\,dv\\
&=\int_0^1\frac{\ln(1+v)}{1+v^2}\,dv-\frac{1}{2}\int_0^1 \frac{\ln(1+v
)}{v}\,dv\\
&=\int_0^1\frac{\ln(1+v)}{1+v^2}\,dv-\frac{1}{4}\int_0^1 \frac{2v\ln(1-v^2)}{v^2}\,dv+\frac{1}{2}\int_0^1 \frac{\ln(1-v)}{v}\,dv\\
\end{align}
In the second integral perform the change of variable $y=v^2$,
\begin{align}\text{R}_3&=\int_0^1\frac{\ln(1+v)}{1+v^2}\,dv+\frac{1}{4}\int_0^1 \frac{\ln(1-v)}{v}\,dv\\
\end{align}
In the second integral perform the change of variable $y=1-v$,
\begin{align}\text{R}_3&=\int_0^1\frac{\ln(1+v)}{1+v^2}\,dv+\frac{1}{4}\int_0^1 \frac{\ln v}{1-v}\,dv\\
&=\int_0^1\frac{\ln(1+v)}{1+v^2}\,dv+\frac{1}{4}\times -\zeta(2)\\
&=\int_0^1\frac{\ln(1+v)}{1+v^2}\,dv-\frac{\pi^2}{24}\\
\end{align}
Perform the change of variable $y=\dfrac{1-v}{1+v}$,
\begin{align}
\text{S}_1&=\int_0^1\frac{\ln(1+v)}{1+v^2}\,dv\\
&=\int_0^1\frac{\ln(\frac{2}{1+v})}{1+v^2}\,dv\\
&=\ln 2\int_0^1 \frac{1}{1+v^2}\,dv-\text{S}_1\\
&=\frac{\pi}{4}\ln 2-\text{S}_1
\end{align}
Therefore,
\begin{align}
\text{S}_1&=\frac{\pi}{8}\ln 2\\
\text{R}_3&=\frac{\pi}{8}\ln 2-\frac{\pi^2}{24}\\
\end{align}
Perform the change of variable $y=\dfrac{1-\sqrt{v}}{1+\sqrt{v}}$,
\begin{align}
\text{R}_4&=\int_0^1\frac{\sqrt{v}\ln\left(\frac{1+v}{2}\right)}{1-v^2}\,dv\\
&=\frac{1}{2}\int_0^1 \frac{(1-v)^2\ln\left(\frac{1+v^2}{(1+v)^2}\right)}{v(1+v^2)}\,dv\\
&=\frac{1}{2}\int_0^1 \frac{(1-v)^2\ln\left(1+v^2\right)}{v(1+v^2)}\,dv+2\text{R}_3\\
&=\frac{1}{2}\int_0^1\frac{\ln(1+v^2)}{v}\,dv-\int_0^1\frac{\ln(1+v^2)}{1+v^2}\,dv+\frac{\pi}{4}\ln 2-\frac{\pi^2}{12}\\
&=\frac{1}{2}\times \frac{1}{4}\zeta(2)-\int_0^1\frac{\ln(1+v^2)}{1+v^2}\,dv+\frac{\pi}{4}\ln 2-\frac{\pi^2}{12}\\
&=\frac{\pi}{4}\ln 2-\frac{\pi^2}{16}-\int_0^1\frac{\ln(1+v^2)}{1+v^2}\,dv\\
&=\frac{\pi}{4}\ln 2-\frac{\pi^2}{16}-\int_0^1\int_0^1\frac{v^2}{(1+v^2)(1+v^2t)}\,dt\,dv\\
&=\frac{\pi}{4}\ln 2-\frac{\pi^2}{16}-\int_0^1 \left[\frac{\arctan\left(v\right)\sqrt{t}-\arctan\left(v\sqrt{t}\right)}{(t-1)\sqrt{t}}\right]_{v=0}^{v=1}\,dt\\
&=\frac{\pi}{4}\ln 2-\frac{\pi^2}{16}-\int_0^1 \frac{\frac{\pi\sqrt{t}}{4}-\arctan\left(\sqrt{t}\right)}{(t-1)\sqrt{t}}\,dt\\
&=\frac{\pi}{4}\ln 2-\frac{\pi^2}{16}+\int_0^1 \frac{\arctan\left(\frac{1-\sqrt{t}}{1+\sqrt{t}}\right)}{(1-t)\sqrt{t}}\,dt-\frac{\pi}{4}\int_0^1 \frac{\sqrt{t}-1}{(t-1)\sqrt{t}}\,dt\\
&=\frac{\pi}{4}\ln 2-\frac{\pi^2}{16}+\int_0^1 \frac{\arctan\left(\frac{1-\sqrt{t}}{1+\sqrt{t}}\right)}{(1-t)\sqrt{t}}\,dt-\frac{\pi}{4}\Big[2\ln\left(1+\sqrt{t}\right)\Big]_0^1\\
&=\int_0^1 \frac{\arctan\left(\frac{1-\sqrt{t}}{1+\sqrt{t}}\right)}{(1-t)\sqrt{t}}\,dt-\frac{\pi}{4}\ln 2-\frac{\pi^2}{16}\\
\end{align}
Perform the change of variable $y=\dfrac{1-\sqrt{t}}{1+\sqrt{t}}$,
\begin{align}
\text{R}_4&=\int_0^1 \frac{\arctan t}{t}\,dt-\frac{\pi}{4}\ln 2-\frac{\pi^2}{16}\\
&=\text{G}-\frac{\pi}{4}\ln 2-\frac{\pi^2}{16}\\
\end{align}
Therefore,
\begin{align}L&=\frac{\pi}{2\sqrt{2}}\text{R}_1+\frac{\pi}{2\sqrt{2}}\left(\frac{\pi}{4}-\frac{1}{2}\ln 2\right) \text{R}_2+\frac{\pi}{2\sqrt{2}}\text{R}_3-\frac{\pi}{4\sqrt{2}}\text{R}_4\\
&=\frac{\pi}{2\sqrt{2}}\left(\frac{\text{G}}{2}-\frac{\pi^2}{32}\right)+\frac{\pi}{2\sqrt{2}}\left(\frac{\pi}{4}-\frac{1}{2}\ln 2\right)^2+\frac{\pi}{2\sqrt{2}}\left(\frac{\pi}{8}\ln 2-\frac{\pi^2}{24}\right)-\\
&\frac{\pi}{4\sqrt{2}}\left(\text{G}-\frac{\pi}{4}\ln 2-\frac{\pi^2}{16}\right)\\
&=\frac{\pi^3}{96\sqrt{2}}+\frac{\pi\ln^2 2}{8\sqrt{2}}
\end{align}
Thus,
\begin{align}\text{I+J}&=\frac{\pi^3}{4\sqrt{2}}-4\text{L}\\
&=\frac{\pi^3}{4\sqrt{2}}-4\left(\frac{\pi^3}{96\sqrt{2}}+\frac{\pi\ln^2 2}{8\sqrt{2}}\right)\\
&=\boxed{\frac{5\pi^3}{24\sqrt{2}}-\frac{\pi\ln^2 2}{2\sqrt{2}}}
\end{align}
Best Answer
Notice $\tan x$ is a periodic function with period $\pi$ and recall following expansion:
$$\frac{1}{\tan x} = \lim_{N\to\infty}\sum_{n=-N}^N \frac{1}{x + n\pi}$$
The integral we seek $$\mathcal{I} \stackrel{def}{=} \int_0^\infty \frac{\sin\tan x}{x} dx = \frac12 \int_{-\infty}^\infty \frac{\sin\tan x}{x} dx = \frac12 \left(\sum_{n=-\infty}^\infty \int_{(n-\frac12)\pi}^{(n+\frac12)\pi}\right)\frac{\sin\tan x}{x}dx $$ can be rewritten as $$ \mathcal{I} = \frac12 \int_{-\frac12\pi}^{\frac12\pi}\sin\tan x\left(\sum_{n=-\infty}^\infty\frac{1}{x+n\pi}\right) dx = \frac12\int_{-\frac12\pi}^{\frac12\pi}\frac{\sin\tan x}{\tan x} dx $$ Change variable to $t = \tan x$, we get
$$\mathcal{I} = \frac12\int_{-\infty}^{\infty} \frac{\sin t}{t(1+t^2)} dt = \frac12\Im\left[\int_{-\infty}^{\infty}\frac{e^{it}-1}{t(1+t^2)} dt\right]$$
We can evaluate the integral on RHS as a contour integral. By completing the contour in upper half-plane and using the fact the integrand has only two poles at $t = \pm i$, we get:
$$\begin{align} \mathcal{I} &= \frac12\Im\left[ 2\pi i \, \mathop{\text{Res}}_{z = i}\left(\frac{e^{it}-1}{t(1+t^2)}\right)\right] = \pi\Re\left[ \frac{e^{i(i)} - 1}{i(i+i)}\right] = \frac{\pi}{2}\left(1 - \frac1e \right)\\ &\approx 0.9929326518994357602762750999834... \end{align} $$
Update
If one don't want to use contour integral, we can replace the last step by a Feymann trick. Consider the function
$$J(a) = \int_0^\infty \frac{\sin(at)}{t(1+t^2)}dt $$
It is easy to see $\mathcal{I} = J(1)$ and $J(a)$ satisfies following ODE for $a > 0$.
$$\left( -\frac{d^2}{da^2} + 1 \right)J(a) = \int_0^\infty \frac{\sin(at)}{t} dt = \int_0^\infty \frac{\sin t}{t} dt = \frac{\pi}{2}$$
This implies $\displaystyle\;J(a) = \frac{\pi}{2} + A e^a + B e^{-a}\;$ for suitably chosen constants $A, B$. Notice $$\begin{align} J(+\infty) &= \lim_{a\to+\infty} \int_0^\infty \frac{\sin t}{t\left(1 + \left(\frac{t}{a}\right)^2\right)} dt = \int_0^\infty \frac{\sin t}{t} dt = \frac{\pi}{2}\\ J'(0^{+}) &= \lim_{a\to 0^{+}} \int_0^\infty \frac{\cos(at)}{1+t^2} dt = \int_0^\infty \frac{dt}{1+t^2} = \frac{\pi}{2} \end{align}$$
This fixes $\;A = 0$, $\displaystyle\;B = -\frac{\pi}{2}\;$ and hence
$$J(a) = \frac{\pi}{2}\left(1 - e^{-a}\right) \quad\implies\quad \mathcal{I} = J(1) = \frac{\pi}{2}\left( 1 - \frac{1}{e}\right)$$