\begin{align}I&=\int_{0}^{\frac{\pi}{2}}\frac{dx}{\left(\sqrt{\sin x}+\sqrt{\cos x}\right)^2}\\
&=\int_{0}^{\frac{\pi}{4}}\frac{dx}{\left(\sqrt{\sin x}+\sqrt{\cos x}\right)^2}
+\int_{\frac{\pi}{4}}^{\frac{\pi}{2}}\frac{dx}{\left(\sqrt{\sin x}+\sqrt{\cos x}\right)^2}\\\end{align}
In the second integral perform the change of variable $y=\frac{\pi}{2}-x$,
\begin{align}I&=2\int_{0}^{\frac{\pi}{4}}\frac{dx}{\left(\sqrt{\sin x}+\sqrt{\cos x}\right)^2}\\
\end{align}
Perform the change of variable $y=\tan x$,
\begin{align}
I&=2\int_{0}^{1}\frac{1}{(1+\sqrt{x})^2\sqrt{1+x^2}}\,dx\\\end{align}
Perform the change of variable $y=\dfrac{1-x}{1+x}$,
\begin{align}I&=\sqrt{2}\int_{0}^{1}\frac{1-\sqrt{1-x^2}}{x^2\sqrt{1+x^2}}\,dx\\
&=-\sqrt{2}\Big[\frac{1-\sqrt{1-x^2}}{x\sqrt{1+x^2}}\Big]_0^1-\sqrt{2}\int_0^1 \frac{\sqrt{1-x^2}-2}{\sqrt{1-x^2}(1+x^2)^{\frac{3}{2}}}\,dx\\
&=-1-\sqrt{2}\int_0^1 \frac{1}{(1+x^2)^{\frac{3}{2}}}\,dx+2\sqrt{2}\int_0^1 \frac{1}{\sqrt{1-x^2}(1+x^2)^{\frac{3}{2}}}\,dx\\
&=-1-\sqrt{2}\left[\frac{x}{\sqrt{1+x^2}}\right]_0^1+2\sqrt{2}\int_0^1 \frac{1}{\sqrt{1-x^2}(1+x^2)^{\frac{3}{2}}}\,dx\\
&=2\sqrt{2}\int_0^1 \frac{1}{\sqrt{1-x^2}(1+x^2)^{\frac{3}{2}}}\,dx-2\\
\end{align}
Perform the change of variable $y=\dfrac{1-x}{1+x}$,
\begin{align}I&=\int_0^1 \frac{x^2+1+2x}{\sqrt{x}(1+x^2)^{\frac32}}\,dx-2\\
&=\int_0^1 \frac{1}{\sqrt{x}\sqrt{1+x^2}}\,dx+2\int_0^1 \frac{\sqrt{x}}{(1+x^2)^{\frac32}}\,dx-2\\
\end{align}
Perform the change of variable $y=\sqrt{x}$ in both integrals,
\begin{align}I&=2\int_0^1 \frac{1}{\sqrt{1+x^4}}\,dx+4\int_0^1 \frac{x^2}{(1+x^4)^{\frac32}}\,dx-2\end{align}
\begin{align}A&=\int_0^1 \frac{1}{\sqrt{1+x^4}}\,dx\end{align}
Perform the change of variable $y=\frac{1}{x}$,
\begin{align}A&=\int_1^\infty \frac{1}{\sqrt{1+x^4}}\,dx=\int_0^\infty \frac{1}{\sqrt{1+x^4}}\,dx-\int_0^1 \frac{1}{\sqrt{1+x^4}}\,dx\\
&=\int_0^\infty \frac{1}{\sqrt{1+x^4}}\,dx-A
\end{align}
Therefore,
\begin{align}A&=\frac{1}{2}\int_0^\infty \frac{1}{\sqrt{1+x^4}}\,dx\end{align}
In the same manner one obtains,
\begin{align}\int_0^1 \frac{x^2}{(1+x^4)^{\frac32}}\,dx&=\frac{1}{2}\int_0^\infty \frac{x^2}{(1+x^4)^{\frac32}}\,dx\end{align}
Therefore,
\begin{align}I&=\int_0^\infty \frac{1}{\sqrt{1+x^4}}\,dx+2\int_0^\infty \frac{x^2}{(1+x^4)^{\frac32}}\,dx-2\end{align}
Perform the change of variable $y=x^4$,
\begin{align}I&=\frac{1}{4}\int_0^\infty \frac{x^{-\frac34}}{(1+x)^{\frac12}}\,dx+\frac{1}{2}\int_0^\infty \frac{x^{-\frac14}}{(1+x)^{\frac32}}\,dx-2\\
&=\frac{1}{4}\text{B}\left(\frac{1}{4},\frac{1}{4}\right)+\frac{1}{2}\text{B}\left(\frac{3}{4},\frac{3}{4}\right)-2\\
&=\frac{1}{4}\times \frac{\Gamma^2\left(\frac{1}{4}\right)}{\Gamma\left(\frac{1}{2}\right)}+\frac{1}{2}\times \frac{\Gamma^2\left(\frac{3}{4}\right)}{\Gamma\left(\frac{3}{2}\right)}-2\\
&=\frac{1}{4}\times \frac{\Gamma^2\left(\frac{1}{4}\right)}{\Gamma\left(\frac{1}{2}\right)}+\frac{1}{2}\times \frac{\Gamma^2\left(\frac{3}{4}\right)}{\frac{1}{2}\Gamma\left(\frac{1}{2}\right)}-2\\
&=\frac{1}{4}\times \frac{\Gamma^2\left(\frac{1}{4}\right)}{\Gamma\left(\frac{1}{2}\right)}+\frac{\Gamma^2\left(\frac{3}{4}\right)}{\Gamma\left(\frac{1}{2}\right)}-2\\
\end{align}
It is well known (Euler's reflection formula) that,
\begin{align}\Gamma\left(\frac{1}{2}\right)=\sqrt{\pi}\end{align}
Therefore,
\begin{align}\boxed{I=\frac{\Gamma^2\left(\frac{1}{4}\right)}{4\sqrt{\pi}}+\frac{\Gamma^2\left(\frac{3}{4}\right)}{\sqrt{\pi}}-2}\end{align}
NB:
$\text{B}$ is the Euler beta function.
Best Answer
Let $\mathcal{I}$ denote the value of the following definite integral:
$$\mathcal{I}:=\int_{0}^{\frac{\pi}{4}}\mathrm{d}x\,\sin{\left(\arccos{\left(\frac{5+\cos{\left(x\right)}}{1+2\cos{\left(x\right)}}\right)}\right)}\approx1.46165\,i.$$
As a commenter pointed out, the integral $\mathcal{I}$ can be evaluated in terms of standard elliptic integrals.
Observing that
$$\sin{\left(\arccos{\left(z\right)}\right)}=\sqrt{1-z^{2}}=i\sqrt{z^{2}-1};~~~\small{z\in\mathbb{R}},$$
and
$$-\frac{2\pi}{3}<x<\frac{2\pi}{3}\implies1<\frac{5+\cos{\left(x\right)}}{1+2\cos{\left(x\right)}},$$
we first reduce the integral to one with an algebraic integrand
$$\begin{align} \mathcal{I} &=\int_{0}^{\frac{\pi}{4}}\mathrm{d}x\,\sin{\left(\arccos{\left(\frac{5+\cos{\left(x\right)}}{1+2\cos{\left(x\right)}}\right)}\right)}\\ &=\int_{0}^{\frac{\pi}{4}}\mathrm{d}x\,\sqrt{1-\left[\frac{5+\cos{\left(x\right)}}{1+2\cos{\left(x\right)}}\right]^{2}}\\ &=i\int_{0}^{\frac{\pi}{4}}\mathrm{d}x\,\sqrt{\left[\frac{5+\cos{\left(x\right)}}{1+2\cos{\left(x\right)}}\right]^{2}-1}\\ &=i\int_{0}^{\frac{\pi}{4}}\mathrm{d}x\,\frac{\sqrt{\left[5+\cos{\left(x\right)}\right]^{2}-\left[1+2\cos{\left(x\right)}\right]^{2}}}{1+2\cos{\left(x\right)}}\\ &=i\int_{0}^{\frac{\pi}{4}}\mathrm{d}x\,\frac{\sqrt{\left[4-\cos{\left(x\right)}\right]\left[6+3\cos{\left(x\right)}\right]}}{1+2\cos{\left(x\right)}}\\ &=i\int_{\frac{1}{\sqrt{2}}}^{1}\mathrm{d}y\,\frac{\sqrt{\left(4-y\right)\left(6+3y\right)}}{\left(1+2y\right)\sqrt{1-y^{2}}};~~~\small{\left[x=\arccos{\left(y\right)}\right]}\\ &=i\int_{\frac{1}{\sqrt{2}}}^{1}\mathrm{d}y\,\frac{\sqrt{3\left(4-y\right)\left(2+y\right)}}{\left(1+2y\right)\sqrt{\left(1-y\right)\left(1+y\right)}}.\\ \end{align}$$
Then, after the right linear fractional transformations, we obtain
$$\begin{align} i^{-1}\mathcal{I} &=\int_{\frac{1}{\sqrt{2}}}^{1}\mathrm{d}y\,\frac{\sqrt{3\left(4-y\right)\left(2+y\right)}}{\left(1+2y\right)\sqrt{\left(1-y\right)\left(1+y\right)}}\\ &=\int_{\frac{1}{\sqrt{2}}}^{1}\mathrm{d}y\,\frac{\sqrt{3\left(\frac{4-y}{2+y}\right)}}{\left(1+2y\right)\sqrt{\left(\frac{1+y}{2+y}\right)\left(\frac{1-y}{2+y}\right)}}\\ &=\int_{\frac{1+\sqrt{2}}{1+2\sqrt{2}}}^{\frac23}\mathrm{d}t\,\frac{\sqrt{3\left(5-6t\right)}}{\left(1-t\right)\left(3t-1\right)\sqrt{t\left(2-3t\right)}};~~~\small{\left[\frac{1+y}{2+y}=t\right]}\\ &=\int_{\frac{3+\sqrt{2}}{7}}^{\frac23}\mathrm{d}t\,\frac{3\left(5-6t\right)}{\left(1-t\right)\left(3t-1\right)\sqrt{3t\left(2-3t\right)\left(5-6t\right)}}\\ &=\int_{\frac{3+\sqrt{2}}{7}}^{\frac23}\mathrm{d}t\,\left[\frac{9}{\left(3t-1\right)}-\frac{1}{\left(1-t\right)}\right]\frac{3}{2\sqrt{3t\left(2-3t\right)\left(5-6t\right)}}\\ &=\int_{\frac{9+3\sqrt{2}}{14}}^{1}\mathrm{d}u\,\left[\frac{3}{\left(2u-1\right)}-\frac{1}{\left(3-2u\right)}\right]\frac{3}{2\sqrt{u\left(1-u\right)\left(5-4u\right)}};~~~\small{\left[t=\frac23u\right]}\\ &=\frac32\int_{\frac{35-15\sqrt{2}}{62}}^{0}\mathrm{d}v\,\frac{(-5)}{\left(5-4v\right)^{2}}\left[\frac{3}{\left(\frac{5-6v}{5-4v}\right)}-\frac{1}{\left(\frac{5-2v}{5-4v}\right)}\right]\frac{\left(5-4v\right)}{5\sqrt{v\left(\frac{1-v}{5-4v}\right)}};~~~\small{\left[u=\frac{5-5v}{5-4v}\right]}\\ &=\frac32\int_{0}^{\frac{35-15\sqrt{2}}{62}}\mathrm{d}v\,\left[\frac{3\left(5-4v\right)}{\left(5-6v\right)}-\frac{\left(5-4v\right)}{\left(5-2v\right)}\right]\frac{1}{\sqrt{v\left(1-v\right)\left(5-4v\right)}}\\ &=\frac32\int_{0}^{\frac{35-15\sqrt{2}}{62}}\mathrm{d}v\,\left[\frac{5}{\left(5-6v\right)}+\frac{5}{\left(5-2v\right)}\right]\frac{1}{\sqrt{v\left(1-v\right)\left(5-4v\right)}}\\ &=3\int_{0}^{\sqrt{\frac{35-15\sqrt{2}}{62}}}\mathrm{d}x\,\left[\frac{5}{\left(5-6x^{2}\right)}+\frac{5}{\left(5-2x^{2}\right)}\right]\frac{1}{\sqrt{\left(1-x^{2}\right)\left(5-4x^{2}\right)}};~~~\small{\left[v=x^{2}\right]}\\ &=\frac{3}{\sqrt{5}}\int_{0}^{\sqrt{\frac{35-15\sqrt{2}}{62}}}\mathrm{d}x\,\left[\frac{1}{\left(1-\frac65x^{2}\right)}+\frac{1}{\left(1-\frac25x^{2}\right)}\right]\frac{1}{\sqrt{\left(1-x^{2}\right)\left(1-\frac45x^{2}\right)}}\\ &=\frac{3}{\sqrt{5}}\left[\Pi{\left(\arcsin{\left(\sqrt{\frac{35-15\sqrt{2}}{62}}\right)},\frac65,\frac45\right)}+\Pi{\left(\arcsin{\left(\sqrt{\frac{35-15\sqrt{2}}{62}}\right)},\frac25,\frac45\right)}\right],\\ \end{align}$$
where $\Pi{(\varphi,n,k)}$ is defined for $0<\varphi<\frac{\pi}{2}\land0<k<1\land0<1-n\sin^{2}{\varphi}$ by
$$\Pi{\left(\varphi,n,k\right)}:=\int_{0}^{\sin{\varphi}}\mathrm{d}x\,\frac{1}{\left(1-nx^{2}\right)\sqrt{\left(1-x^{2}\right)\left(1-k^{2}x^{2}\right)}}.$$