Your feature collection is private, so I can't apply your code. But, here you have how to mask clouds from Landsat 5 Collection 1 Tier 2 TOA Reflectance and compute NDVI:
var region = /* color: #0b4a8b */ee.Geometry.Polygon(
[[[-71.9000244140625, -30.817346256492073],
[-70.7080078125, -30.826780904779778],
[-70.6640625, -30.372875188118016],
[-71.8670654296875, -30.410781790845878]]]),
landsat = ee.ImageCollection("LANDSAT/LT05/C01/T2_TOA");
var collection = landsat.filterBounds(region).filterDate('2000-01-01', '2000-02-28');
var getQABits = function(image, start, end, newName) {
// Compute the bits we need to extract.
var pattern = 0;
for (var i = start; i <= end; i++) {
pattern += Math.pow(2, i);
}
// Return a single band image of the extracted QA bits, giving the band
// a new name.
return image.select([0], [newName])
.bitwiseAnd(pattern)
.rightShift(start);
};
// A function to mask out cloudy pixels.
var cloud_shadows = function(image) {
// Select the QA band.
var QA = image.select(['BQA']);
// Get the internal_cloud_algorithm_flag bit.
return getQABits(QA, 7,8, 'Cloud_shadows');
// Return an image masking out cloudy areas.
};
// A function to mask out cloudy pixels.
var clouds = function(image) {
// Select the QA band.
var QA = image.select(['BQA']);
// Get the internal_cloud_algorithm_flag bit.
return getQABits(QA, 4,4, 'Cloud').neq(1);
// Return an image masking out cloudy areas.
};
var maskClouds = function(image) {
var cs = cloud_shadows(image);
var c = clouds(image);
image = image.updateMask(cs);
return image.updateMask(c);
};
var NDVI = function(image) {
return image.normalizedDifference(['B5', 'B4']).rename('NDVI');
};
var ndvi = collection.map(maskClouds).map(NDVI);
Map.addLayer(collection, {bands: ['B4', 'B3', 'B2'],min: [0,0,0],max: [0.2, 0.2, 0.2]}, 'Landsat with clouds');
Map.addLayer(ndvi,{min: -1, max: 1, palette: ['blue', 'white', 'green']}, 'NDVI cloud free');
Link to reproducible example: https://code.earthengine.google.com/2602e7e0f67bd31e1f53379dcd326148
You need to check Landsat documentation to know how to decode BQA from this collection.
Best Answer
What I ended up doing:
1) Download and decompress your landsat tile. Do not open and look at the images in ArcMap because ArcMap will create .proj on the fly and this will confuse poor fmask and prevent it from running.
2) Once your landsat tile of choice's file is decompressed/unzipped/etc. Follow the directions on fmask's website for masking clouds. I used their stand alone program so I didn't have to download Matlab.
3) After fmask runs there will be a file titled xxxfmask in with all the bands for the landsat imagry.
4) This file can only be opened in ENVI NOT in ArcGIS
5) I was lucky enough to have access to ENVI so I loaded the files onto one of my school's computers and opened them in ENVI and converted them to .shp files
6) Do whatever you want with the .shp cloud masks in Arcmap