[Tex/LaTex] additional curly braces around matrix

bracesmatricestikz-pgf

I would like to have matrix like this one:
enter image description here

I know how to make color frames in tikz, but I havee problems with braces.
I tried this code:

\documentclass[a4paper,10pt]{article}
\usepackage[utf8]{inputenc}

\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amsthm}
\usepackage{mathtools} %do robienia mini macierzy
%           linki w spisie tresci
\usepackage[customcolors]{hf-tikz} % do kolorowych macierzy
\usetikzlibrary{patterns}
\usetikzlibrary{decorations.pathreplacing}



\newcommand{\tikzmark}[1]{\tikz[overlay,remember picture,baseline=(#1.base)]
  \node (#1) {\strut};}

%opening
\title{}
\author{}

\begin{document}
\begin{equation}
  \left[ \begin{array}{cccccc}
    v_{1,1}  &  v_{1,2} & \tikzmark{F1p} v_{1,3}  \tikzmark{F1k} & \tikzmark{F2p} \textcolor{white}{v_{1,4}}  \tikzmark{F2k}&  &   \\
    v_{2,1}  & v_{2,2} & v_{2,3} & & &   \\
    \tikzmark{E1p} v_{3,1}  & v_{3,2} & v_{3,3} & v_{3,4}   & &   \\
     & &  v_{4,3} & v_{4,4} & v_{4,5}  & v_{4,6} \\
    \tikzmark{E1k} & & & v_{5,4} & v_{5,5} & v_{5,6}   \\
    & &  & v_{6,4} & v_{6,5} & v_{6,6} \\
  \end{array}\right]
\end{equation}



\begin{tikzpicture}[overlay, remember picture,decoration={brace,amplitude=2pt}]
\draw[decorate,thick] (F1p.north) -- (F1k.north)
      node [midway,above=5pt] {$F'$};
\draw[decorate,thick] (F2p.north) -- (F2k.north)
      node [midway,above=5pt] {$F''$};

\draw [decorate,thick,xshift=-2pt,yshift=0pt]
(E1p) -- (E1k) node [black,midway,xshift=-0.6cm] 
{\footnotesize $P_1$};
\end{tikzpicture}

\end{document}

but it gives the following result:
enter image description here

I only know a bit how to make upper braces.
How should I make it properly?

I tried to use the code:

\documentclass[a4paper,10pt]{article}
\usepackage[utf8]{inputenc}

\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amsthm}
\usepackage{mathtools} %do robienia mini macierzy
%           linki w spisie tresci
\usepackage[customcolors]{hf-tikz} % do kolorowych macierzy
\usetikzlibrary{patterns}
\usetikzlibrary{matrix,decorations.pathreplacing,calc}



\newcommand{\tikzmark}[1]{\tikz[overlay,remember picture,baseline=(#1.base)]
  \node (#1) {\strut};}

%opening
\title{}
\author{}

\pgfkeys{tikz/mymatrixenv/.style={decoration=brace,every left delimiter/.style={xshift=3pt},every right delimiter/.style={xshift=-3pt}}}
\pgfkeys{tikz/mymatrix/.style={matrix of math nodes,left delimiter=[,right delimiter={]},inner sep=2pt,column sep=1em,row sep=0.5em,nodes={inner sep=0pt}}}
\pgfkeys{tikz/mymatrixbrace/.style={decorate,thick}}
\newcommand\mymatrixbraceoffseth{0.5em}
\newcommand\mymatrixbraceoffsetv{0.2em}


\newcommand*\mymatrixbraceright[4][m]{
    \draw[mymatrixbrace] ($(#1.north west)!(#1-#3-1.south west)!(#1.south west)-(\mymatrixbraceoffseth,0)$)
        -- node[left=2pt] {#4} 
        ($(#1.north west)!(#1-#2-1.north west)!(#1.south west)-(\mymatrixbraceoffseth,0)$);
}
\newcommand*\mymatrixbraceleft[4][m]{
    \draw[mymatrixbrace] ($(#1.north east)!(#1-#2-1.north east)!(#1.south east)+(\mymatrixbraceoffseth,0)$)
        -- node[right=2pt] {#4} 
        ($(#1.north east)!(#1-#3-1.south east)!(#1.south east)+(\mymatrixbraceoffseth,0)$);
}
\newcommand*\mymatrixbracetop[4][m]{
    \draw[mymatrixbrace] ($(#1.north west)!(#1-1-#2.north west)!(#1.north east)+(0,\mymatrixbraceoffsetv)$)
        -- node[above=2pt] {#4} 
        ($(#1.north west)!(#1-1-#3.north east)!(#1.north east)+(0,\mymatrixbraceoffsetv)$);
}
\newcommand*\mymatrixbracebottom[4][m]{
    \draw[mymatrixbrace] ($(#1.south west)!(#1-1-#3.south east)!(#1.south east)-(0,\mymatrixbraceoffsetv)$)
        -- node[below=2pt] {#4} 
        ($(#1.south west)!(#1-1-#2.south west)!(#1.south east)-(0,\mymatrixbraceoffsetv)$);
}

\begin{document}



\begin{equation}
\mathbf{X} = 
\begin{tikzpicture}[baseline=0cm,mymatrixenv]
    \matrix [mymatrix,inner sep=4pt,row sep=1em] (m)  
    {
    v_{1,1}  &  v_{1,2} & \tikzmark{F1p} v_{1,3}  \tikzmark{F1k} & \tikzmark{F2p} \textcolor{white}{v_{1,4}}  \tikzmark{F2k}&  &   \\
    v_{2,1}  & v_{2,2} & v_{2,3} & & &   \\
    \tikzmark{E1p} v_{3,1}  & v_{3,2} & v_{3,3} & v_{3,4}   & &   \\
     & &  v_{4,3} & v_{4,4} & v_{4,5}  & v_{4,6} \\
    \tikzmark{E1k} & & & v_{5,4} & v_{5,5} & v_{5,6}   \\
    & &  & v_{6,4} & v_{6,5} & v_{6,6} \\
    };

    % Braces     
    \mymatrixbraceright{1}{3}{$B'$}
    \mymatrixbraceright{4}{6}{$B''$}
%    \mymatrixbracetop{1}{3}{$C'$}
%    \mymatrixbracetop{4}{6}{$C''$}
%    \mymatrixbracebottom{3}{3}{$F'$}
%    \mymatrixbracebottom{4}{4}{$F''$}
\end{tikzpicture}
\end{equation}



\end{document}

but it gives the error: "package pgf error: no shape named m-4-1 is known \mymatrixbraceright{4}{6}{$B''$}"

works much better, after changing corresponding line to:

\pgfkeys{tikz/mymatrix/.style={matrix of math nodes, nodes in empty cells, left delimiter=[,right delimiter={]},inner sep=2pt,column sep=1em,row sep=0.5em,nodes={inner sep=0pt}}}

but still there is a gap. How to delete the gap or make it smaller?
enter image description here

Best Answer

An (maybe simpler) alternative to @Milo's excellent answer, I've given all nodes a minimum size.

enter image description here

I also changed the \mymatrixbrace<side> commands slightly so you don't need the calc library anymore.

\documentclass[a4paper,10pt]{article}
\usepackage[utf8]{inputenc}

\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amsthm}
\usepackage{mathtools} %do robienia mini macierzy
%           linki w spisie tresci
\usepackage[customcolors]{hf-tikz} % do kolorowych macierzy
\usetikzlibrary{patterns}
\usetikzlibrary{matrix,decorations.pathreplacing}

%opening
\title{}
\author{}

\pgfkeys{tikz/mymatrixenv/.style={decoration={brace},every left delimiter/.style={xshift=8pt},every right delimiter/.style={xshift=-8pt}}}
\pgfkeys{tikz/mymatrix/.style={matrix of math nodes,nodes in empty cells,left delimiter={[},right delimiter={]},inner sep=1pt,outer sep=1.5pt,column sep=2pt,row sep=2pt,nodes={minimum width=20pt,minimum height=10pt,anchor=center,inner sep=0pt,outer sep=0pt}}}
\pgfkeys{tikz/mymatrixbrace/.style={decorate,thick}}

\newcommand*\mymatrixbraceright[4][m]{
    \draw[mymatrixbrace] (#1.west|-#1-#3-1.south west) -- node[left=2pt] {#4} (#1.west|-#1-#2-1.north west);
}
\newcommand*\mymatrixbraceleft[4][m]{
    \draw[mymatrixbrace] (#1.east|-#1-#2-1.north east) -- node[right=2pt] {#4} (#1.east|-#1-#2-1.south east);
}
\newcommand*\mymatrixbracetop[4][m]{
    \draw[mymatrixbrace] (#1.north-|#1-1-#2.north west) -- node[above=2pt] {#4} (#1.north-|#1-1-#3.north east);
}
\newcommand*\mymatrixbracebottom[4][m]{
    \draw[mymatrixbrace] (#1.south-|#1-1-#2.north east) -- node[below=2pt] {#4} (#1.south-|#1-1-#3.north west);
}

\begin{document}

\begin{equation}
    \mathbf{X} = 
    \begin{tikzpicture}[baseline={-0.5ex},mymatrixenv]
        \matrix [mymatrix,inner sep=4pt] (m)  
        {
        v_{1,1} & v_{1,2} & v_{1,3} &         &         &         \\
        v_{2,1} & v_{2,2} & v_{2,3} &         &         &         \\
        v_{3,1} & v_{3,2} & v_{3,3} & v_{3,4} &         &         \\
                &         & v_{4,3} & v_{4,4} & v_{4,5} & v_{4,6} \\
                &         &         & v_{5,4} & v_{5,5} & v_{5,6} \\
                &         &         & v_{6,4} & v_{6,5} & v_{6,6} \\
        };

        % Braces     
        \mymatrixbraceright{1}{3}{$B'$}
        \mymatrixbraceright{4}{6}{$B''$}
        \mymatrixbracetop{1}{3}{$C'$}
        \mymatrixbracetop{4}{6}{$C''$}
        \mymatrixbracebottom{3}{3}{$F'$}
        \mymatrixbracebottom{4}{4}{$F''$}
        \mymatrixbraceleft{3}{3}{$E'$}
        \mymatrixbraceleft{4}{4}{$E''$}
    \end{tikzpicture}
\end{equation}



\end{document}